From 882db61ad7b9045eb5e2cf963b849633b9b67a03 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 21 Dec 2024 16:14:18 -0500 Subject: [PATCH 01/11] fix cgroup all Signed-off-by: drfaust92 --- autogen/main/cluster.tf.tmpl | 3 ++- cluster.tf | 3 ++- modules/beta-private-cluster-update-variant/cluster.tf | 3 ++- modules/beta-private-cluster/cluster.tf | 3 ++- modules/beta-public-cluster-update-variant/cluster.tf | 3 ++- modules/beta-public-cluster/cluster.tf | 3 ++- modules/private-cluster-update-variant/cluster.tf | 3 ++- modules/private-cluster/cluster.tf | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index d997ac407b..4c12984f3c 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1111,6 +1111,7 @@ resource "google_container_node_pool" "windows_pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : {cgroup = local.node_pools_cgroup_mode[each.value["name"]]} )) != 0 ? [1] : [] @@ -1119,7 +1120,7 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } {% endif %} diff --git a/cluster.tf b/cluster.tf index d53799e500..c364a2909b 100644 --- a/cluster.tf +++ b/cluster.tf @@ -804,6 +804,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -812,7 +813,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 3fec97d43a..4c7158d3ff 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -975,6 +975,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -983,7 +984,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 73d12ec042..b7a0f46cea 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -894,6 +894,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -902,7 +903,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 057e419081..5907579ca7 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -945,6 +945,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -953,7 +954,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 42a350b6ba..32504c7868 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -864,6 +864,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -872,7 +873,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index fa03099a0c..83a26adb1e 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -914,6 +914,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -922,7 +923,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 3ca8117ae6..99f1a88d8e 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -834,6 +834,7 @@ resource "google_container_node_pool" "pools" { for_each = length(merge( local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]], + local.node_pools_cgroup_mode["all"] == "" ? {} : { cgroup = local.node_pools_cgroup_mode["all"] }, local.node_pools_cgroup_mode[each.value["name"]] == "" ? {} : { cgroup = local.node_pools_cgroup_mode[each.value["name"]] } )) != 0 ? [1] : [] @@ -842,7 +843,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } From f0e941c4615e748ae73688db73466aa2e0e24b61 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 26 Dec 2024 14:20:50 -0500 Subject: [PATCH 02/11] attmept cgroup fix Signed-off-by: drfaust92 --- README.md | 159 --------------- autogen/main/cluster.tf.tmpl | 2 +- autogen/main/variables_defaults.tf.tmpl | 17 +- .../beta-autopilot-private-cluster/README.md | 130 ------------- .../beta-autopilot-public-cluster/README.md | 122 ------------ .../README.md | 182 ------------------ .../variables_defaults.tf | 3 +- modules/beta-private-cluster/README.md | 182 ------------------ .../variables_defaults.tf | 3 +- .../README.md | 174 ----------------- .../variables_defaults.tf | 3 +- modules/beta-public-cluster/README.md | 174 ----------------- .../beta-public-cluster/variables_defaults.tf | 3 +- .../private-cluster-update-variant/README.md | 167 ---------------- .../variables_defaults.tf | 3 +- modules/private-cluster/README.md | 167 ---------------- modules/private-cluster/variables_defaults.tf | 3 +- .../safer-cluster-update-variant/README.md | 113 ----------- modules/safer-cluster/README.md | 113 ----------- variables_defaults.tf | 3 +- 20 files changed, 24 insertions(+), 1699 deletions(-) diff --git a/README.md b/README.md index ea612e0a92..aae006062a 100644 --- a/README.md +++ b/README.md @@ -134,165 +134,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 4c12984f3c..06f30c19c2 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1120,7 +1120,7 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) } } {% endif %} diff --git a/autogen/main/variables_defaults.tf.tmpl b/autogen/main/variables_defaults.tf.tmpl index 8d771e2dc5..2dcd601584 100644 --- a/autogen/main/variables_defaults.tf.tmpl +++ b/autogen/main/variables_defaults.tf.tmpl @@ -128,14 +128,15 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) - node_pools_cgroup_mode = merge( - { all = "" }, - { default-node-pool = "" }, - zipmap( - [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] - ), - var.node_pools_cgroup_mode + + node_pools_cgroup_mode = merge( + { all = "" }, + { default-node-pool = "" }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_cgroup_mode ) } {% endif %} diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 888c450654..285835a638 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -67,136 +67,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `true` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_variant | (Optional) The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX\_THROUGHPUT. | `string` | `null` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 729039e281..fab9183581 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -62,128 +62,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `true` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_variant | (Optional) The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX\_THROUGHPUT. | `string` | `null` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 45d57d1651..c8a59c281e 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -164,188 +164,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/beta-private-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 30e38fb25f..6467db93a0 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -142,188 +142,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/beta-private-cluster/variables_defaults.tf +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index abdc91ce57..166dfea4e6 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,180 +159,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-public-cluster-update-variant/variables_defaults.tf b/modules/beta-public-cluster-update-variant/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/beta-public-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-public-cluster-update-variant/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 2b2789a5ce..f880bfcb4d 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,180 +137,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/beta-public-cluster/variables_defaults.tf +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 7a2b611385..ea8f1040b9 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -161,173 +161,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/private-cluster-update-variant/variables_defaults.tf +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 49d64bad5f..714a857fe3 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -139,173 +139,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/modules/private-cluster/variables_defaults.tf +++ b/modules/private-cluster/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index ee0e3c39e8..772dff12d4 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -197,119 +197,6 @@ For simplicity, we suggest using `roles/container.admin` and ``` -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | -| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster | `string` | n/a | yes | -| network | The VPC network to host the cluster in | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `string` | n/a | yes | -| region | The region to host the cluster in | `string` | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | -| zones | The zones to host the cluster in | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| enable\_mesh\_certificates | Mesh certificate configuration value | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - To provision this example, run the following from within this directory: diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ee0e3c39e8..772dff12d4 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -197,119 +197,6 @@ For simplicity, we suggest using `roles/container.admin` and ``` -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | -| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster | `string` | n/a | yes | -| network | The VPC network to host the cluster in | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `string` | n/a | yes | -| region | The region to host the cluster in | `string` | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | -| zones | The zones to host the cluster in | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| enable\_mesh\_certificates | Mesh certificate configuration value | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - To provision this example, run the following from within this directory: diff --git a/variables_defaults.tf b/variables_defaults.tf index c33f3828e7..9b8a43ba14 100644 --- a/variables_defaults.tf +++ b/variables_defaults.tf @@ -127,12 +127,13 @@ locals { ), var.node_pools_linux_node_configs_sysctls ) + node_pools_cgroup_mode = merge( { all = "" }, { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : ""] + [for node_pool in var.node_pools : {}] ), var.node_pools_cgroup_mode ) From ce811afa6f8c5fe569ff544f09409b80dd80e68c Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 26 Dec 2024 15:28:26 -0500 Subject: [PATCH 03/11] attmept cgroup fix Signed-off-by: drfaust92 --- README.md | 159 +++++++++++++++ .../beta-autopilot-private-cluster/README.md | 130 +++++++++++++ .../beta-autopilot-public-cluster/README.md | 122 ++++++++++++ .../README.md | 182 ++++++++++++++++++ modules/beta-private-cluster/README.md | 182 ++++++++++++++++++ .../README.md | 174 +++++++++++++++++ modules/beta-public-cluster/README.md | 174 +++++++++++++++++ .../private-cluster-update-variant/README.md | 167 ++++++++++++++++ modules/private-cluster/README.md | 167 ++++++++++++++++ .../safer-cluster-update-variant/README.md | 113 +++++++++++ modules/safer-cluster/README.md | 113 +++++++++++ 11 files changed, 1683 insertions(+) diff --git a/README.md b/README.md index aae006062a..ea612e0a92 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,165 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 285835a638..888c450654 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -67,6 +67,136 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `true` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_variant | (Optional) The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX\_THROUGHPUT. | `string` | `null` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index fab9183581..729039e281 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -62,6 +62,128 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `true` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_variant | (Optional) The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX\_THROUGHPUT. | `string` | `null` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index c8a59c281e..45d57d1651 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -164,6 +164,188 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6467db93a0..30e38fb25f 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -142,6 +142,188 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 166dfea4e6..abdc91ce57 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,6 +159,180 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index f880bfcb4d..2b2789a5ce 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,6 +137,180 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index ea8f1040b9..7a2b611385 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -161,6 +161,173 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 714a857fe3..49d64bad5f 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -139,6 +139,173 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 772dff12d4..ee0e3c39e8 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -197,6 +197,119 @@ For simplicity, we suggest using `roles/container.admin` and ``` +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster | `string` | n/a | yes | +| network | The VPC network to host the cluster in | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| zones | The zones to host the cluster in | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| enable\_mesh\_certificates | Mesh certificate configuration value | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + To provision this example, run the following from within this directory: diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 772dff12d4..ee0e3c39e8 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -197,6 +197,119 @@ For simplicity, we suggest using `roles/container.admin` and ``` +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster | `string` | n/a | yes | +| network | The VPC network to host the cluster in | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| zones | The zones to host the cluster in | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| enable\_mesh\_certificates | Mesh certificate configuration value | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + To provision this example, run the following from within this directory: From a1b7da1400d14d73ea95afeba57862ebe180e2a2 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Thu, 26 Dec 2024 16:52:54 -0500 Subject: [PATCH 04/11] attmept cgroup fix Signed-off-by: drfaust92 --- autogen/main/cluster.tf.tmpl | 2 +- autogen/main/variables_defaults.tf.tmpl | 2 +- cluster.tf | 2 +- examples/node_pool/main.tf | 5 +++++ modules/beta-private-cluster-update-variant/cluster.tf | 2 +- .../variables_defaults.tf | 2 +- modules/beta-private-cluster/cluster.tf | 2 +- modules/beta-private-cluster/variables_defaults.tf | 2 +- modules/beta-public-cluster-update-variant/cluster.tf | 2 +- .../beta-public-cluster-update-variant/variables_defaults.tf | 2 +- modules/beta-public-cluster/cluster.tf | 2 +- modules/beta-public-cluster/variables_defaults.tf | 2 +- modules/private-cluster-update-variant/cluster.tf | 2 +- modules/private-cluster-update-variant/variables_defaults.tf | 2 +- modules/private-cluster/cluster.tf | 2 +- modules/private-cluster/variables_defaults.tf | 2 +- variables_defaults.tf | 2 +- 17 files changed, 21 insertions(+), 16 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 06f30c19c2..c94381b594 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1120,7 +1120,7 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } {% endif %} diff --git a/autogen/main/variables_defaults.tf.tmpl b/autogen/main/variables_defaults.tf.tmpl index 2dcd601584..0348cc8c1f 100644 --- a/autogen/main/variables_defaults.tf.tmpl +++ b/autogen/main/variables_defaults.tf.tmpl @@ -134,7 +134,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/cluster.tf b/cluster.tf index c364a2909b..02fe8edbe0 100644 --- a/cluster.tf +++ b/cluster.tf @@ -813,7 +813,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 3713311de6..fa4a26cf96 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -150,4 +150,9 @@ module "gke" { "net.core.netdev_max_backlog" = "20000" } } + + node_pools_cgroup_mode = { + all = "CGROUP_MODE_UNSPECIFIED" + pool-01 = "CGROUP_MODE_V2" + } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 4c7158d3ff..4d3eda03f8 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -984,7 +984,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/beta-private-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index b7a0f46cea..1ddecda15b 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -903,7 +903,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/beta-private-cluster/variables_defaults.tf +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 5907579ca7..cc55bb8498 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -954,7 +954,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/beta-public-cluster-update-variant/variables_defaults.tf b/modules/beta-public-cluster-update-variant/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/beta-public-cluster-update-variant/variables_defaults.tf +++ b/modules/beta-public-cluster-update-variant/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 32504c7868..04f1fa27be 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -873,7 +873,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/beta-public-cluster/variables_defaults.tf +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 83a26adb1e..188ae916a2 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -923,7 +923,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/private-cluster-update-variant/variables_defaults.tf +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 99f1a88d8e..b72c613956 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -843,7 +843,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) == "" ? null : coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"]) + cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] } } diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/modules/private-cluster/variables_defaults.tf +++ b/modules/private-cluster/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) diff --git a/variables_defaults.tf b/variables_defaults.tf index 9b8a43ba14..5c7f51b42f 100644 --- a/variables_defaults.tf +++ b/variables_defaults.tf @@ -133,7 +133,7 @@ locals { { default-node-pool = "" }, zipmap( [for node_pool in var.node_pools : node_pool["name"]], - [for node_pool in var.node_pools : {}] + [for node_pool in var.node_pools : ""] ), var.node_pools_cgroup_mode ) From bcb2a6c839f7d40d6c464f97611b6c76be3c7ca0 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 28 Dec 2024 09:34:02 -0500 Subject: [PATCH 05/11] CR comments Signed-off-by: drfaust92 --- examples/node_pool/main.tf | 2 +- .../node_pool/testdata/TestNodePool.json | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index fa4a26cf96..a15ba27e46 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -152,7 +152,7 @@ module "gke" { } node_pools_cgroup_mode = { - all = "CGROUP_MODE_UNSPECIFIED" + all = "CGROUP_MODE_V1" pool-01 = "CGROUP_MODE_V2" } } diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json index 413d0ce050..8d8dd5a310 100644 --- a/test/integration/node_pool/testdata/TestNodePool.json +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -158,7 +158,7 @@ "nodeConfig": { "diskSizeGb": 100, "diskType": "pd-balanced", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "gcfsConfig": {}, "imageType": "COS_CONTAINERD", "loggingConfig": { @@ -205,7 +205,7 @@ "config": { "diskSizeGb": 100, "diskType": "pd-balanced", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "gcfsConfig": {}, "imageType": "COS_CONTAINERD", "loggingConfig": { @@ -270,7 +270,7 @@ "config": { "diskSizeGb": 100, "diskType": "pd-standard", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "imageType": "COS_CONTAINERD", "machineType": "e2-medium", "metadata": { @@ -425,7 +425,7 @@ ], "diskSizeGb": 30, "diskType": "pd-standard", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "imageType": "COS_CONTAINERD", "labels": { "all-pools-example": "true", @@ -505,7 +505,7 @@ "config": { "diskSizeGb": 100, "diskType": "pd-standard", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "imageType": "COS_CONTAINERD", "kubeletConfig": { "cpuCfsQuota": true, @@ -603,7 +603,7 @@ "config": { "diskSizeGb": 100, "diskType": "pd-standard", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "imageType": "COS_CONTAINERD", "labels": { "all-pools-example": "true", @@ -698,7 +698,7 @@ }, "diskSizeGb": 100, "diskType": "pd-standard", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", "imageType": "COS_CONTAINERD", "labels": { "all-pools-example": "true", From 4b6745fc629572cc743c7c364a0a17152a0f1459 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 28 Dec 2024 20:38:58 -0500 Subject: [PATCH 06/11] CR comments Signed-off-by: drfaust92 --- test/integration/node_pool/testdata/TestNodePool.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json index 8d8dd5a310..2175d01c60 100644 --- a/test/integration/node_pool/testdata/TestNodePool.json +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -158,7 +158,7 @@ "nodeConfig": { "diskSizeGb": 100, "diskType": "pd-balanced", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", "gcfsConfig": {}, "imageType": "COS_CONTAINERD", "loggingConfig": { @@ -205,7 +205,7 @@ "config": { "diskSizeGb": 100, "diskType": "pd-balanced", - "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V1", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", "gcfsConfig": {}, "imageType": "COS_CONTAINERD", "loggingConfig": { From b3ce28bce7a43c3863e843f3a6d4ed4b9329f628 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sun, 29 Dec 2024 12:36:55 -0500 Subject: [PATCH 07/11] CR comments Signed-off-by: drfaust92 --- autogen/main/cluster.tf.tmpl | 2 +- cluster.tf | 2 +- .../README.md | 182 ------------------ .../cluster.tf | 2 +- modules/beta-private-cluster/README.md | 182 ------------------ modules/beta-private-cluster/cluster.tf | 2 +- .../README.md | 174 ----------------- .../cluster.tf | 2 +- modules/beta-public-cluster/README.md | 174 ----------------- modules/beta-public-cluster/cluster.tf | 2 +- .../private-cluster-update-variant/README.md | 167 ---------------- .../private-cluster-update-variant/cluster.tf | 2 +- modules/private-cluster/README.md | 167 ---------------- modules/private-cluster/cluster.tf | 2 +- .../safer-cluster-update-variant/README.md | 113 ----------- modules/safer-cluster/README.md | 113 ----------- 16 files changed, 8 insertions(+), 1280 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index c94381b594..f54358547a 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1120,7 +1120,7 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } {% endif %} diff --git a/cluster.tf b/cluster.tf index 02fe8edbe0..f80f4f7e69 100644 --- a/cluster.tf +++ b/cluster.tf @@ -813,7 +813,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 45d57d1651..c8a59c281e 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -164,188 +164,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 4d3eda03f8..da22de20c1 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -984,7 +984,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 30e38fb25f..6467db93a0 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -142,188 +142,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 1ddecda15b..bad56c1e2b 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -903,7 +903,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index abdc91ce57..166dfea4e6 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,180 +159,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index cc55bb8498..a4ff6242c4 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -954,7 +954,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 2b2789a5ce..f880bfcb4d 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,180 +137,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 04f1fa27be..69ef80bb71 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -873,7 +873,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 7a2b611385..ea8f1040b9 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -161,173 +161,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 188ae916a2..3e0c6999f6 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -923,7 +923,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 49d64bad5f..714a857fe3 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -139,173 +139,6 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | -| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | -| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | -| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | -| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | -| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | -| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | -| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | -| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | -| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | -| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | -| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | -| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | -| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster (required) | `string` | n/a | yes | -| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | -| network\_policy | Enable network policy addon | `bool` | `false` | no | -| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | -| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | -| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | -| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | -| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | -| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | -| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | -| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | -| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | -| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | -| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| fleet\_membership | Fleet membership (if registered) | -| gateway\_api\_channel | The gateway api channel of this cluster. | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity pool | -| identity\_service\_enabled | Whether Identity Service is enabled | -| instance\_group\_urls | List of GKE generated instance groups | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| mesh\_certificates\_config | Mesh certificates configuration | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | -| zones | List of zones in which the cluster resides | - ## node_pools variable diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index b72c613956..fb93cc2677 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -843,7 +843,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode["all"] == "" ? local.node_pools_cgroup_mode[each.value["name"]] == "" ? null : local.node_pools_cgroup_mode[each.value["name"]] : local.node_pools_cgroup_mode["all"] + cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] } } diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index ee0e3c39e8..772dff12d4 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -197,119 +197,6 @@ For simplicity, we suggest using `roles/container.admin` and ``` -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | -| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster | `string` | n/a | yes | -| network | The VPC network to host the cluster in | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `string` | n/a | yes | -| region | The region to host the cluster in | `string` | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | -| zones | The zones to host the cluster in | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| enable\_mesh\_certificates | Mesh certificate configuration value | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - To provision this example, run the following from within this directory: diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ee0e3c39e8..772dff12d4 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -197,119 +197,6 @@ For simplicity, we suggest using `roles/container.admin` and ``` -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | -| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | -| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | -| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | -| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | -| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | -| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | -| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| description | The description of the cluster | `string` | `""` | no | -| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | -| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | -| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | -| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | -| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | -| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | -| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | -| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | -| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | -| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | -| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | -| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | -| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | -| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | -| istio | (Beta) Enable Istio addon | `bool` | `false` | no | -| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | -| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | -| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | -| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | -| name | The name of the cluster | `string` | n/a | yes | -| network | The VPC network to host the cluster in | `string` | n/a | yes | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | -| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | -| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | -| project\_id | The project ID to host the cluster in | `string` | n/a | yes | -| region | The region to host the cluster in | `string` | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | -| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | -| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | -| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | -| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | -| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | -| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | -| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | -| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | -| zones | The zones to host the cluster in | `list(string)` | `[]` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cluster\_id | Cluster ID | -| enable\_mesh\_certificates | Mesh certificate configuration value | -| endpoint | Cluster endpoint | -| endpoint\_dns | Cluster endpoint DNS | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | Node pool versions by node pool name | -| peering\_name | The name of the peering between this cluster and the Google owned VPC. | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - To provision this example, run the following from within this directory: From 8a99082b0ccd955224d3ebf338581e60c1ecd418 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sun, 29 Dec 2024 12:42:24 -0500 Subject: [PATCH 08/11] CR comments Signed-off-by: drfaust92 --- .../README.md | 182 ++++++++++++++++++ modules/beta-private-cluster/README.md | 182 ++++++++++++++++++ .../README.md | 174 +++++++++++++++++ modules/beta-public-cluster/README.md | 174 +++++++++++++++++ .../private-cluster-update-variant/README.md | 167 ++++++++++++++++ .../safer-cluster-update-variant/README.md | 113 +++++++++++ modules/safer-cluster/README.md | 113 +++++++++++ 7 files changed, 1105 insertions(+) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index c8a59c281e..45d57d1651 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -164,6 +164,188 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6467db93a0..30e38fb25f 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -142,6 +142,188 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 166dfea4e6..abdc91ce57 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -159,6 +159,180 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index f880bfcb4d..2b2789a5ce 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,6 +137,180 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| cluster\_telemetry\_type | Available options include ENABLED, DISABLED, and SYSTEM\_ONLY | `string` | `null` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kalm\_config | (Beta) Whether KALM is enabled for this cluster. | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Sets which mode of auditing should be used for the cluster's workloads. Accepted values are DISABLED, BASIC. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Sets which mode to use for Protect workload vulnerability scanning feature. Accepted values are DISABLED, BASIC. | `string` | `""` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index ea8f1040b9..7a2b611385 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -161,6 +161,173 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 772dff12d4..ee0e3c39e8 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -197,6 +197,119 @@ For simplicity, we suggest using `roles/container.admin` and ``` +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster | `string` | n/a | yes | +| network | The VPC network to host the cluster in | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| zones | The zones to host the cluster in | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| enable\_mesh\_certificates | Mesh certificate configuration value | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + To provision this example, run the following from within this directory: diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 772dff12d4..ee0e3c39e8 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -197,6 +197,119 @@ For simplicity, we suggest using `roles/container.admin` and ``` +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. If set then also set var.create\_service\_account to false to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `string` | `""` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. Explicitly set to false if var.compute\_engine\_service\_account is set to avoid 'value depends on resource attributes that cannot be determined until apply' errors. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `ADVANCED_DATAPATH` enables Dataplane-V2 feature. `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation as a fallback since upgrading to V2 requires a cluster re-creation. | `string` | `"ADVANCED_DATAPATH"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| dns\_cache | (Beta) The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | `bool` | `false` | no | +| enable\_private\_endpoint | When true, the cluster's private endpoint is used as the cluster endpoint and access through the public endpoint is disabled. When false, either endpoint can be used. This field only applies to private clusters, when enable\_private\_nodes is true | `bool` | `true` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | `bool` | `true` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | (Beta) Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gke\_backup\_agent\_config | (Beta) Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | `bool` | `true` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | `bool` | `true` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| istio | (Beta) Enable Istio addon | `bool` | `false` | no | +| istio\_auth | (Beta) The authentication type between services in Istio. | `string` | `"AUTH_MUTUAL_TLS"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | `string` | `null` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster | `string` | n/a | yes | +| network | The VPC network to host the cluster in | `string` | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| node\_pools | List of maps containing node pools | `list(map(string))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| project\_id | The project ID to host the cluster in | `string` | n/a | yes | +| region | The region to host the cluster in | `string` | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` role is assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing node pools | `list(map(string))` | `[]` | no | +| workload\_config\_audit\_mode | (beta) Workload config audit mode. | `string` | `"DISABLED"` | no | +| workload\_vulnerability\_mode | (beta) Vulnerability mode. | `string` | `""` | no | +| zones | The zones to host the cluster in | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| enable\_mesh\_certificates | Mesh certificate configuration value | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + To provision this example, run the following from within this directory: From 70d9890c099d88aa9ab71638996fc8c7a121e08a Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sun, 29 Dec 2024 12:49:51 -0500 Subject: [PATCH 09/11] CR comments Signed-off-by: drfaust92 --- modules/private-cluster/README.md | 167 ++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 714a857fe3..49d64bad5f 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -139,6 +139,173 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| add\_cluster\_firewall\_rules | Create additional firewall rules | `bool` | `false` | no | +| add\_master\_webhook\_firewall\_rules | Create master\_webhook firewall rules for ports defined in `firewall_inbound_ports` | `bool` | `false` | no | +| add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | +| additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | +| additive\_vpc\_scope\_dns\_domain | This will enable Cloud DNS additive VPC scope. Must provide a domain name that is unique within the VPC. For this to work cluster\_dns = `CLOUD_DNS` and cluster\_dns\_scope = `CLUSTER_SCOPE` must both be set as well. | `string` | `""` | no | +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                     = bool
    autoscaling_profile         = string
    min_cpu_cores               = number
    max_cpu_cores               = number
    min_memory_gb               = number
    max_memory_gb               = number
    gpu_resources               = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair                 = bool
    auto_upgrade                = bool
    disk_size                   = optional(number)
    disk_type                   = optional(string)
    image_type                  = optional(string)
    strategy                    = optional(string)
    max_surge                   = optional(number)
    max_unavailable             = optional(number)
    node_pool_soak_duration     = optional(string)
    batch_soak_duration         = optional(string)
    batch_percentage            = optional(number)
    batch_node_count            = optional(number)
    enable_secure_boot          = optional(bool, false)
    enable_integrity_monitoring = optional(bool, true)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enable_integrity_monitoring": true,
  "enable_secure_boot": false,
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | +| cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | +| cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | +| config\_connector | Whether ConfigConnector is enabled for this cluster. | `bool` | `false` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | +| datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | +| deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| description | The description of the cluster | `string` | `""` | no | +| disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | +| dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `false` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | `bool` | `false` | no | +| enable\_cilium\_clusterwide\_network\_policy | Enable Cilium Cluster Wide Network Policies on the cluster | `bool` | `false` | no | +| enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | +| enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | +| enable\_default\_node\_pools\_metadata | Whether to enable the default node pools metadata key-value pairs such as `cluster_name` and `node_pool` | `bool` | `true` | no | +| enable\_gcfs | Enable image streaming on cluster level. | `bool` | `false` | no | +| enable\_identity\_service | (Optional) Enable the Identity Service component, which allows customers to use external identity providers with the K8S API. | `bool` | `false` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | `bool` | `false` | no | +| enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | +| enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | +| enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `true` | no | +| enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | +| enable\_secret\_manager\_addon | Enable the Secret Manager add-on for this cluster | `bool` | `false` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | +| enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `false` | no | +| filestore\_csi\_driver | The status of the Filestore CSI driver addon, which allows the usage of filestore instance as volumes | `bool` | `false` | no | +| firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | +| firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | +| fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | +| gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | +| gcp\_public\_cidrs\_access\_enabled | Allow access through Google Cloud public IP addresses | `bool` | `null` | no | +| gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | +| gke\_backup\_agent\_config | Whether Backup for GKE agent is enabled for this cluster. | `bool` | `false` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | +| http\_load\_balancing | Enable httpload balancer addon | `bool` | `true` | no | +| identity\_namespace | The workload pool to attach all Kubernetes service accounts to. (Default value of `enabled` automatically sets project-based pool `[project_id].svc.id.goog`) | `string` | `"enabled"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | `number` | `0` | no | +| insecure\_kubelet\_readonly\_port\_enabled | Whether or not to set `insecure_kubelet_readonly_port_enabled` for node pool defaults and autopilot clusters. Note: this can be set at the node pool level separately within `node_pools`. | `bool` | `null` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | `bool` | `false` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | `string` | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | `string` | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | `string` | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | `string` | `"latest"` | no | +| logging\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, CONTROLLER\_MANAGER, KCP\_CONNECTION, KCP\_SSHD, SCHEDULER, and WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | `string` | `"logging.googleapis.com/kubernetes"` | no | +| maintenance\_end\_time | Time window specified for recurring maintenance operations in RFC3339 format | `string` | `""` | no | +| maintenance\_exclusions | List of maintenance exclusions. A cluster can have up to three | `list(object({ name = string, start_time = string, end_time = string, exclusion_scope = string }))` | `[]` | no | +| maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | +| maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | +| master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | +| master\_ipv4\_cidr\_block | (Optional) The IP range in CIDR notation to use for the hosted master network. | `string` | `null` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, APISERVER, SCHEDULER, CONTROLLER\_MANAGER, STORAGE, HPA, POD, DAEMONSET, DEPLOYMENT, STATEFULSET, KUBELET, CADVISOR and DCGM. In beta provider, WORKLOADS is supported on top of those 12 values. (WORKLOADS is deprecated and removed in GKE 1.24.) KUBELET and CADVISOR are only supported in GKE 1.29.3-gke.1093000 and above. Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | +| name | The name of the cluster (required) | `string` | n/a | yes | +| network | The VPC network to host the cluster in (required) | `string` | n/a | yes | +| network\_policy | Enable network policy addon | `bool` | `false` | no | +| network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | +| node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | +| node\_pools\_cgroup\_mode | Map of strings containing cgroup node config by node-pool name | `map(string)` | 
{
  "all": "",
  "default-node-pool": ""
}
| no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_linux\_node\_configs\_sysctls | Map of maps containing linux node config sysctls by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | `map(list(string))` | 
{
  "all": [
    "https://www.googleapis.com/auth/cloud-platform"
  ],
  "default-node-pool": []
}
| no | +| node\_pools\_resource\_labels | Map of maps containing resource labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_resource\_manager\_tags | Map of maps containing resource manager tags by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | `map(list(string))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | `map(list(object({ key = string, value = string, effect = string })))` | 
{
  "all": [],
  "default-node-pool": []
}
| no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | +| notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | +| notification\_filter\_event\_type | Choose what type of notifications you want to receive. If no filters are applied, you'll receive all notification types. Can be used to filter what notifications are sent. Accepted values are UPGRADE\_AVAILABLE\_EVENT, UPGRADE\_EVENT, and SECURITY\_BULLETIN\_EVENT. | `list(string)` | `[]` | no | +| private\_endpoint\_subnetwork | The subnetwork to use for the hosted master network. | `string` | `null` | no | +| project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | +| ray\_operator\_config | The Ray Operator Addon configuration for this cluster. | 
object({
    enabled            = bool
    logging_enabled    = optional(bool, false)
    monitoring_enabled = optional(bool, false)
  })
| 
{
  "enabled": false,
  "logging_enabled": false,
  "monitoring_enabled": false
}
| no | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | `string` | `null` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | `bool` | `true` | no | +| registry\_project\_ids | Projects holding Google Container Registries. If empty, we use the cluster project. If a service account is created and the `grant_registry_access` variable is set to `true`, the `storage.objectViewer` and `artifactregsitry.reader` roles are assigned on these projects. | `list(string)` | `[]` | no | +| release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | +| resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | +| security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | +| service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | +| service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | +| shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | +| shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | +| stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | +| subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | +| timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | `list(string)` | `[]` | no | +| windows\_node\_pools | List of maps containing Windows node pools | `list(map(string))` | `[]` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | `list(string)` | `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | +| endpoint | Cluster endpoint | +| endpoint\_dns | Cluster endpoint DNS | +| fleet\_membership | Fleet membership (if registered) | +| gateway\_api\_channel | The gateway api channel of this cluster. | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity pool | +| identity\_service\_enabled | Whether Identity Service is enabled | +| instance\_group\_urls | List of GKE generated instance groups | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation used for the hosted master network | +| master\_version | Current master kubernetes version | +| mesh\_certificates\_config | Mesh certificates configuration | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | Node pool versions by node pool name | +| peering\_name | The name of the peering between this cluster and the Google owned VPC. | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| secret\_manager\_addon\_enabled | Whether Secret Manager add-on is enabled | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| tpu\_ipv4\_cidr\_block | The IP range in CIDR notation used for the TPUs | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether vertical pod autoscaling enabled | +| zones | List of zones in which the cluster resides | + ## node_pools variable From 5574df3952e2f1f610601c9fa3bb965d422072ad Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Wed, 1 Jan 2025 19:22:13 -0500 Subject: [PATCH 10/11] Update cluster.tf Co-authored-by: Andrew Peabody --- cluster.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster.tf b/cluster.tf index f80f4f7e69..45eae4c367 100644 --- a/cluster.tf +++ b/cluster.tf @@ -813,7 +813,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"], local.node_pools_cgroup_mode["all"], null) } } From 7f0f2c20256dd2e9b786edae5cc56536ced0423b Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 1 Jan 2025 19:32:27 -0500 Subject: [PATCH 11/11] CR comments Signed-off-by: drfaust92 --- autogen/main/cluster.tf.tmpl | 2 +- cluster.tf | 2 +- modules/beta-private-cluster-update-variant/cluster.tf | 2 +- modules/beta-private-cluster/cluster.tf | 2 +- modules/beta-public-cluster-update-variant/cluster.tf | 2 +- modules/beta-public-cluster/cluster.tf | 2 +- modules/private-cluster-update-variant/cluster.tf | 2 +- modules/private-cluster/cluster.tf | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index f54358547a..6261a06406 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -1120,7 +1120,7 @@ resource "google_container_node_pool" "windows_pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } {% endif %} diff --git a/cluster.tf b/cluster.tf index 45eae4c367..c241aef605 100644 --- a/cluster.tf +++ b/cluster.tf @@ -813,7 +813,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"], local.node_pools_cgroup_mode["all"], null) + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index da22de20c1..f86cd27458 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -984,7 +984,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index bad56c1e2b..2a2362e60d 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -903,7 +903,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index a4ff6242c4..2fd7580755 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -954,7 +954,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 69ef80bb71..ae1d831ccb 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -873,7 +873,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 3e0c6999f6..104b69aed6 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -923,7 +923,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } } diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index fb93cc2677..c81cf14264 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -843,7 +843,7 @@ resource "google_container_node_pool" "pools" { local.node_pools_linux_node_configs_sysctls["all"], local.node_pools_linux_node_configs_sysctls[each.value["name"]] ) - cgroup_mode = local.node_pools_cgroup_mode[each.value["name"]] == "" ? local.node_pools_cgroup_mode["all"] == "" ? null : local.node_pools_cgroup_mode["all"] : local.node_pools_cgroup_mode[each.value["name"]] + cgroup_mode = coalesce(local.node_pools_cgroup_mode[each.value["name"]], local.node_pools_cgroup_mode["all"], null) } }