Staging Lets Encrypt Certificates working 🎉🎉🎉 (#10)

coilysiren · Nov 22, 2023 · 46b1c5f · 46b1c5f
1 parent d82ea7d
commit 46b1c5f
Show file tree

Hide file tree

Showing 16 changed files with 512 additions and 115 deletions.
diff --git a/README.md b/README.md
@@ -86,9 +86,74 @@ $ make upgrade
 
 ## Deployment
 
-This deployment command assumes you are locally authenticated to both gcloud and kubectl. Directions on how to do so are out of scope for this documentation. Please consult your team's local deployment tooling and instructions!
+This deployment command assumes you are locally installed to gcloud and kubectl, in addition to all the other above installations
+
+Note that, during the deploy process, you will likely need to enable several google APIs. Do so when prompted, then run the deploy again. This will show up as messages like:
+
+> googleapi: Error 403: $API has not been used in project $PROECT before or it is disabled. Enable it by visiting...
+
+These instructions all assume you are starting from the top level directory, whenever a `$SHELL` command is given.
+
+### 0. Name your project
+
+Open `config.yml` and modify the "Project configuration" section
+
+### 1. Create a new project
+
+Create a new project via https://console.cloud.google.com/, then set its name in `config.yml`
+
+```yaml
+# config.yml
+project: dotted-hope-405813
+```
+
+Then you should run the following commands once, replacing $PROJECT with your actual project ID.
 
 ```bash
-source ./venv/bin/activate
-invoke deploy # see tasks.py for source code
+# $SHELL
+$ gcloud config set project $PROJECT
+$ gcloud auth application-default login --project $PROJECT
+```
+
+### 2. Create a terraform state bucket
+
+Create a terraform state bucket via https://console.cloud.google.com/, then set its name in `config.yml`
+
+```yaml
+# config.yml
+bucket: coilysiren-k8s-gpc-tfstate-3
+```
+
+Then you must set its name manually in every `state.tf` file. Open every `state.tf` file in the repo. You will see a block like this:
+
+```hcl
+terraform {
+ backend "gcs" {
+ bucket = "coilysiren-k8s-gpc-tfstate-3"
+ prefix = "terraform/state"
+ }
+}
+```
+
+You should modify the `bucket = ...` line with your bucket name, same as in `config.yml`.
+
+Finally, import you import the bucket into terraform.
+
+```bash
+# $SHELL
+$ cd infrastructure/foundation/
+$ terraform init
+$ terraform import google_storage_bucket.default coilysiren-k8s-gpc-tfstate-3
+```
+
+Note that, when you deploy in the next step, you might have to modify the state bucket's region. The goal is to avoid replacing the state bucket.
+
+### 3. Deploy
+
+Run the deploy script
+
+```bash
+# $SHELL
+$ source ./venv/bin/activate
+$ invoke deploy # see tasks.py for source code
 ```
diff --git a/config.yml b/config.yml
@@ -1,3 +1,13 @@
-name: gke-test-2
-project: root-territory-384205
+# Project configuration
+name: gke-test-3
+domain: gke-test-3.coilysiren.me
+zone: coilysiren.me # domain - name = zone
+email: [email protected]
+
+# Google Cloud Platform configuration
+project: dotted-hope-405813
+statebucket: coilysiren-k8s-gpc-tfstate-3
 region: us-central1
+
+# https://github.com/cert-manager/cert-manager/releases
+cert-manager-version: v1.13.2
diff --git a/infrastructure/application/.terraform.lock.hcl b/infrastructure/application/.terraform.lock.hcl
diff --git a/infrastructure/application/main.tf b/infrastructure/application/main.tf
@@ -1,27 +1,88 @@
 locals {
- name = yamldecode(file("../../config.yml")).name
+ zone = yamldecode(file("../../config.yml")).zone
+ domain = yamldecode(file("../../config.yml")).domain
+ email = yamldecode(file("../../config.yml")).email
 }
 
-# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config
-data "google_client_config" "default" {}
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_user
+resource "aws_iam_user" "user" {
+ name = "certbot"
+}
+
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_access_key
+resource "aws_iam_access_key" "key" {
+ user = aws_iam_user.user.name
+}
+
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy
+resource "aws_iam_policy" "policy" {
+ name = "certbot"
+
+ # https://cert-manager.io/docs/configuration/acme/dns01/route53/
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Effect = "Allow",
+ Action = "route53:ChangeResourceRecordSets",
+ Resource = data.aws_route53_zone.zone.arn,
+ },
+ {
+ Effect = "Allow"
+ Action = [
+ "route53:Get*",
+ "route53:List*",
+ ]
+ Resource = "*"
+ },
+ ]
+ })
+}
+
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_user_policy_attachment
+resource "aws_iam_user_policy_attachment" "test-attach" {
+ user = aws_iam_user.user.name
+ policy_arn = aws_iam_policy.policy.arn
+}
 
-data "kubernetes_service" "service" {
+# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret
+resource "kubernetes_secret" "secret" {
  metadata {
- name = "application"
+ name = "route53-credentials-secret"
+ }
+
+ data = {
+ "access-key-id" = aws_iam_access_key.key.id
+ "secret-access-key" = aws_iam_access_key.key.secret
+ }
+}
+
+# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret
+resource "kubernetes_secret" "cert" {
+ metadata {
+ name = "web-ssl"
+ }
+ type = "kubernetes.io/tls"
+ data = {
+ "tls.crt" = ""
+ "tls.key" = ""
+ }
+ lifecycle {
+ ignore_changes = [
+ metadata,
+ data,
+ ]
  }
 }
 
 # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone
 data "aws_route53_zone" "zone" {
- name = "coilysiren.me."
+ name = "${local.zone}."
  private_zone = false
 }
 
-# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record
-resource "aws_route53_record" "record" {
- zone_id = data.aws_route53_zone.zone.zone_id
- name = "${local.name}.coilysiren.me."
- type = "A"
- ttl = "300"
- records = [data.kubernetes_service.service.status.0.load_balancer.0.ingress.0.ip]
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address
+resource "google_compute_address" "address" {
+ name = "application-ingress"
+ address_type = "EXTERNAL"
 }
diff --git a/infrastructure/application/state.tf b/infrastructure/application/state.tf
@@ -1,16 +1,23 @@
 terraform {
  backend "gcs" {
- bucket = "coilysiren-k8s-gpc-tfstate-0"
+ bucket = "coilysiren-k8s-gpc-tfstate-3"
  prefix = "terraform/state/application"
  }
 }
 
+locals {
+ statebucket = yamldecode(file("../../config.yml")).statebucket
+}
+
 # https://registry.terraform.io/providers/hashicorp/google/latest/docs
 provider "google" {
  project = yamldecode(file("../../config.yml")).project
  region = yamldecode(file("../../config.yml")).region
 }
 
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config
+data "google_client_config" "default" {}
+
 # https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs
 provider "kubernetes" {
  host = "https://${data.terraform_remote_state.foundation.outputs.endpoint}"
@@ -25,13 +32,10 @@ provider "aws" {
  region = "us-east-1"
 }
 
-# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/project
-data "google_project" "default" {}
-
 data "terraform_remote_state" "foundation" {
  backend = "gcs"
  config = {
- bucket = "coilysiren-k8s-gpc-tfstate-0"
+ bucket = local.statebucket
  prefix = "terraform/state"
  }
 }
diff --git a/infrastructure/deployment.yml b/infrastructure/deployment.yml
@@ -0,0 +1,38 @@
+apiVersion: v1
+items:
+ - apiVersion: v1
+ kind: Service
+ metadata:
+ annotations:
+ cloud.google.com/neg: '{"ingress": true}'
+ name: application
+ spec:
+ ports:
+ - name: http
+ port: 80
+ targetPort: 8080
+ selector:
+ app: application
+ type: ClusterIP
+ - apiVersion: apps/v1
+ kind: Deployment
+ metadata:
+ name: application
+ spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: application
+ template:
+ metadata:
+ labels:
+ app: application
+ spec:
+ containers:
+ - image: us-central1-docker.pkg.dev/dotted-hope-405813/repository/gke-test-3:certs-add8cea-kai
+ name: application
+ ports:
+ - containerPort: 8080
+kind: List
+metadata:
+ resourceVersion: ""
diff --git a/infrastructure/foundation/main.tf b/infrastructure/foundation/main.tf
@@ -2,9 +2,6 @@ locals {
  name = yamldecode(file("../../config.yml")).name
 }
 
-# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config
-data "google_client_config" "default" {}
-
 # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_service_account
 resource "google_service_account" "gke" {
  account_id = local.name
@@ -64,6 +61,28 @@ resource "google_project_iam_binding" "iamserviceAccountUser" {
  ]
 }
 
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam
+# https://cloud.google.com/logging/docs/access-control
+resource "google_project_iam_binding" "logWriter" {
+ project = data.google_client_config.default.project
+ role = "roles/logging.logWriter"
+
+ members = [
+ "serviceAccount:${data.google_project.default.number}@cloudservices.gserviceaccount.com",
+ ]
+}
+
+# https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/google_project_iam
+# https://cloud.google.com/monitoring/access-control
+resource "google_project_iam_binding" "metricWriter" {
+ project = data.google_client_config.default.project
+ role = "roles/monitoring.metricWriter"
+
+ members = [
+ "serviceAccount:${data.google_project.default.number}@cloudservices.gserviceaccount.com",
+ ]
+}
+
 # https://registry.terraform.io/modules/terraform-google-modules/network/google/latest
 module "vpc" {
  source = "terraform-google-modules/network/google"
@@ -111,19 +130,18 @@ module "vpc" {
 
 # https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/latest
 module "gke" {
- name = local.name
- source = "terraform-google-modules/kubernetes-engine/google"
- project_id = data.google_client_config.default.project
- region = data.google_client_config.default.region
- network = module.vpc.network_name
- subnetwork = module.vpc.subnets_names[0]
- zones = ["${data.google_client_config.default.region}-a"] # default is every zone, we only want one for $$$ reasons
- remove_default_node_pool = true
- deletion_protection = false
- default_max_pods_per_node = 16
- initial_node_count = 1
- ip_range_pods = "pods-range"
- ip_range_services = "services-range"
+ name = local.name
+ source = "terraform-google-modules/kubernetes-engine/google"
+ project_id = data.google_client_config.default.project
+ region = data.google_client_config.default.region
+ network = module.vpc.network_name
+ subnetwork = module.vpc.subnets_names[0]
+ zones = ["${data.google_client_config.default.region}-a"] # default is every zone, we only want one for $$$ reasons
+ remove_default_node_pool = true
+ deletion_protection = false
+ initial_node_count = 1
+ ip_range_pods = "pods-range"
+ ip_range_services = "services-range"
  # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool
  # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_node_config
  node_pools = [