From ea9378c695dd45dd0ee6ddbad0c8582d7293fdf3 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Tue, 23 Jul 2024 14:12:00 -0500 Subject: [PATCH 1/9] Create PostgreSQL Secret even if postgresql.enabled=false Ensures that the PostgreSQL Secret is created, even if `postgresql.enabled=false`. This ensures that we support a use case where folks want to use an external instance of PostgreSQL, but still want the Secret to automatically be generated with the proper connection string. With a recent change, we would skip creation of this secret if PostgreSQL was disabled which forced users to create a Secret themselves. This change now allows them to continue providing the `auth` values and letting the chart build the Secret with the correct connection string. Closes https://github.com/PrefectHQ/prefect-helm/issues/358 --- charts/prefect-server/README.md | 8 +++++++- charts/prefect-server/README.md.gotmpl | 8 +++++++- charts/prefect-server/templates/secret.yaml | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index adeadfd6..08cd5982 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -45,9 +45,15 @@ prefect-server: postgresql: # Disable the objects from the bundled PostgreSQL chart enabled: false + auth: - # Provide the name of an existing secret following the instructions above. + # Option 1: provide the name of an existing secret following the instructions above. existingSecret: + + # Option 2: provide the connection string details directly + username: myuser + password: mypass + database: mydb ``` Two secrets are created when not providing an existing secret name: diff --git a/charts/prefect-server/README.md.gotmpl b/charts/prefect-server/README.md.gotmpl index d8594b57..34e835de 100644 --- a/charts/prefect-server/README.md.gotmpl +++ b/charts/prefect-server/README.md.gotmpl @@ -44,9 +44,15 @@ prefect-server: postgresql: # Disable the objects from the bundled PostgreSQL chart enabled: false + auth: - # Provide the name of an existing secret following the instructions above. + # Option 1: provide the name of an existing secret following the instructions above. existingSecret: + + # Option 2: provide the connection string details directly + username: myuser + password: mypass + database: mydb ``` Two secrets are created when not providing an existing secret name: diff --git a/charts/prefect-server/templates/secret.yaml b/charts/prefect-server/templates/secret.yaml index 462b59d8..d5ffe4f4 100644 --- a/charts/prefect-server/templates/secret.yaml +++ b/charts/prefect-server/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.postgresql.enabled (not .Values.postgresql.auth.existingSecret) }} +{{- if not .Values.postgresql.auth.existingSecret }} apiVersion: v1 kind: Secret metadata: From 315e929b6de388aab206b424075d4d44ae2f4469 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 09:27:04 -0500 Subject: [PATCH 2/9] Remove custom config from postgresql subchart The `postgresql` key in `values.yaml` is for overriding the PostgreSQL chart's default values. It's not a good place to define custom keys because these are conflated with the actual configuration from the subchart. This changes removes those custom configs. It also removes any configured values that already matched the default from the PostgreSQL subchart, and were therefore not doing anything. --- charts/prefect-server/README.md | 4 - charts/prefect-server/values.schema.json | 109 ----------------------- charts/prefect-server/values.yaml | 19 ---- 3 files changed, 132 deletions(-) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index 08cd5982..721b0986 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -139,16 +139,12 @@ No secrets are created when providing an existing secret. | namespaceOverride | string | `""` | fully override common.names.namespace | | postgresql.auth.database | string | `"server"` | name for a custom database | | postgresql.auth.enablePostgresUser | bool | `false` | determines whether an admin user is created within postgres | -| postgresql.auth.existingSecret | string | `""` | Name of existing secret to use for PostgreSQL credentials. | | postgresql.auth.password | string | `"prefect-rocks"` | password for the custom user. Ignored if `auth.existingSecret` with key `password` is provided | | postgresql.auth.username | string | `"prefect"` | name for a custom user | -| postgresql.containerPorts | object | `{"postgresql":5432}` | PostgreSQL container port | | postgresql.enabled | bool | `true` | enable use of bitnami/postgresql subchart | -| postgresql.externalHostname | string | `""` | | | postgresql.image.tag | string | `"14.3.0"` | Version tag, corresponds to tags at https://hub.docker.com/r/bitnami/postgresql/ | | postgresql.primary.initdb.user | string | `"postgres"` | specify the PostgreSQL username to execute the initdb scripts | | postgresql.primary.persistence.enabled | bool | `false` | enable PostgreSQL Primary data persistence using PVC | -| postgresql.primary.persistence.size | string | `"8Gi"` | PVC Storage Request for PostgreSQL volume | | server.affinity | object | `{}` | affinity for server pods assignment | | server.autoscaling.enabled | bool | `false` | enable autoscaling for server | | server.autoscaling.maxReplicas | int | `100` | maximum number of server replicas | diff --git a/charts/prefect-server/values.schema.json b/charts/prefect-server/values.schema.json index 9320ab94..cd519ab5 100644 --- a/charts/prefect-server/values.schema.json +++ b/charts/prefect-server/values.schema.json @@ -613,115 +613,6 @@ } } }, - "postgresql": { - "type": "object", - "title": "PostgreSQL", - "description": "Postgresql configuration", - "properties": { - "enabled": { - "type": "boolean", - "title": "Enabled", - "description": "enable postgresql" - }, - "auth": { - "type": "object", - "title": "Auth", - "description": "postgresql authentication configuration", - "properties": { - "enablePostgresUser": { - "type": "boolean", - "title": "Enable Postgres User", - "description": "determines whether an admin user is created within postgres" - }, - "database": { - "type": "string", - "title": "Database", - "description": "name for a custom database" - }, - "username": { - "type": "string", - "title": "Username", - "description": "name for a custom user" - }, - "password": { - "type": "string", - "title": "Password", - "description": "password for the custom user" - }, - "existingSecret": { - "type": "string", - "title": "Existing Secret", - "description": "name of an existing secret containing the postgresql password" - } - } - }, - "containerPorts": { - "type": "object", - "title": "Container Ports", - "description": "PostgreSQL container port", - "properties": { - "postgresql": { - "type": ["integer","string"], - "title": "Postgres", - "description": "PostgreSQL container port" - } - } - }, - "externalHostname": { - "type": "string", - "title": "External Hostname", - "description": "external hostname for the postgresql service" - }, - "primary": { - "type": "object", - "title": "Primary", - "description": "Initdb configuration", - "properties": { - "initdb": { - "type": "object", - "title": "Initdb", - "description": "Initdb configuration", - "properties": { - "user": { - "type": "string", - "title": "User", - "description": "specify the PostgreSQL username to execute the initdb scripts" - } - } - }, - "persistence": { - "type": "object", - "title": "Persistence", - "description": "Primary persistence configuration", - "properties": { - "enabled": { - "type": "boolean", - "title": "Enabled", - "description": "enable PostgreSQL Primary data persistence using PVC" - }, - "size": { - "type": "string", - "title": "Size", - "description": "PVC Storage Request for PostgreSQL volume" - } - } - }, - "image": { - "type": "object", - "title": "Image", - "description": "Postgres image configuration", - "properties": { - "tag": { - "type": "string", - "title": "Tag", - "description": " Version tag, corresponds to tags at https://hub.docker.com/r/bitnami/postgresql/" - } - } - } - } - } - } - }, "common": { "type": "object", "title": "Common", diff --git a/charts/prefect-server/values.yaml b/charts/prefect-server/values.yaml index e10992ba..402c34ad 100644 --- a/charts/prefect-server/values.yaml +++ b/charts/prefect-server/values.yaml @@ -310,23 +310,6 @@ postgresql: # -- password for the custom user. Ignored if `auth.existingSecret` with key `password` is provided password: prefect-rocks - ## This secret must contain two key-value pairs where the first key is `connection-string` and the value is the - ## connection string containing your password (e.g. postgresql+asyncpg://{username}:{password}@{hostname}/{database}). - ## The second key-value pair has the key `password` and the value is the {password} used in the connection string - # -- Name of existing secret to use for PostgreSQL credentials. - existingSecret: "" - - # -- PostgreSQL container port - containerPorts: - postgresql: 5432 - - # externalHostname defines the address to contact an externally - # managed postgres database instance at. This is not required if - # `internalPostgres` is `true` - externalHostname: "" - - ## postgresql configuration below here is only used if using the subchart - ## Initdb configuration ## ref: https://github.com/bitnami/containers/tree/main/bitnami/postgresql#specifying-initdb-arguments primary: @@ -340,8 +323,6 @@ postgresql: persistence: # -- enable PostgreSQL Primary data persistence using PVC enabled: false - # -- PVC Storage Request for PostgreSQL volume - size: 8Gi image: # -- Version tag, corresponds to tags at https://hub.docker.com/r/bitnami/postgresql/ From cf95c2ac9eb22b2becfe4d52c3ffcd44c983164c Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 10:11:18 -0500 Subject: [PATCH 3/9] Support providing external connection string When the PostgreSQL subchart is disabled, we need a way to provide the connection string information for the external instance. This adds the required values and template helpers to calculate the connection string in this scenario. --- charts/prefect-server/README.md | 5 ++ charts/prefect-server/templates/_helpers.tpl | 80 +++++++++++++++++--- charts/prefect-server/values.yaml | 15 +++- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index 721b0986..82b50a24 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -145,6 +145,11 @@ No secrets are created when providing an existing secret. | postgresql.image.tag | string | `"14.3.0"` | Version tag, corresponds to tags at https://hub.docker.com/r/bitnami/postgresql/ | | postgresql.primary.initdb.user | string | `"postgres"` | specify the PostgreSQL username to execute the initdb scripts | | postgresql.primary.persistence.enabled | bool | `false` | enable PostgreSQL Primary data persistence using PVC | +| secret.database | string | `""` | database for the PostgreSQL connection string | +| secret.host | string | `""` | host for the PostgreSQL connection string | +| secret.password | string | `""` | password for the PostgreSQL connection string | +| secret.port | string | `""` | port for the PostgreSQL connection string | +| secret.username | string | `""` | username for the PostgreSQL connection string | | server.affinity | object | `{}` | affinity for server pods assignment | | server.autoscaling.enabled | bool | `false` | enable autoscaling for server | | server.autoscaling.maxReplicas | int | `100` | maximum number of server replicas | diff --git a/charts/prefect-server/templates/_helpers.tpl b/charts/prefect-server/templates/_helpers.tpl index 61db3667..2281a879 100644 --- a/charts/prefect-server/templates/_helpers.tpl +++ b/charts/prefect-server/templates/_helpers.tpl @@ -9,20 +9,80 @@ Create the name of the service account to use {{- end -}} {{- end -}} +// ----- Connection string templates ------ + {{/* server.postgres-hostname: Generate the hostname of the postgresql service If a subchart is used, evaluate using its fullname function - as {subchart.fullname}-{namespace} + and append the namespace at the end. Otherwise, the configured external hostname will be returned */}} {{- define "server.postgres-hostname" -}} {{- if .Values.postgresql.enabled -}} - {{- $subchart_overrides := .Values.postgresql -}} - {{- $name := include "postgresql.v1.primary.fullname" (dict "Values" $subchart_overrides "Chart" (dict "Name" "postgresql") "Release" .Release) -}} - {{- printf "%s.%s" $name .Release.Namespace -}} +{{- $subchart_overrides := .Values.postgresql -}} +{{- $name := include "postgresql.v1.primary.fullname" (dict "Values" $subchart_overrides "Chart" (dict "Name" "postgresql") "Release" .Release) -}} +{{- printf "%s.%s" $name .Release.Namespace -}} +{{- else -}} +{{- .Values.secret.host | required ".Values.secret.host is required." -}} +{{- end -}} +{{- end -}} + +{{/* + server.postgres-port: + Generate the port of the postgresql service + If a subchart is used, evaluate using its port function + Otherwise, the configured port will be returned +*/}} +{{- define "server.postgres-port" -}} +{{- if .Values.postgresql.enabled -}} +{{- $subchart_overrides := .Values.postgresql -}} +{{- include "postgresql.v1.service.port" (dict "Values" $subchart_overrides) -}} +{{- else -}} +{{- .Values.secret.port | required ".Values.secret.port is required." -}} +{{- end -}} +{{- end -}} + +{{/* + server.postgres-username: + Generate the username for postgresql + If a subchart is used, evaluate using its username function + Otherwise, the configured username will be returned +*/}} +{{- define "server.postgres-username" -}} +{{- if .Values.postgresql.enabled -}} +{{- $subchart_overrides := .Values.postgresql -}} +{{- include "postgresql.v1.username" (dict "Values" $subchart_overrides) -}} {{- else -}} - {{- .Values.postgresql.externalHostname -}} +{{- .Values.secret.username | required ".Values.secret.username is required." -}} +{{- end -}} +{{- end -}} + +{{/* + server.postgres-password: + Generate the password for postgresql + If a subchart is used, evaluate using its password value + Otherwise, the configured password will be returned +*/}} +{{- define "server.postgres-password" -}} +{{- if .Values.postgresql.enabled -}} +{{- .Values.postgresql.auth.password | required ".Values.postgresql.auth.password is required." -}} +{{- else -}} +{{- .Values.secret.password | required ".Values.secret.password is required." -}} +{{- end -}} +{{- end -}} + +{{/* + server.postgres-database: + Generate the database for postgresql + If a subchart is used, evaluate using its database value + Otherwise, the configured database will be returned +*/}} +{{- define "server.postgres-database" -}} +{{- if .Values.postgresql.enabled -}} +{{- .Values.postgresql.auth.database | required ".Values.postgresql.auth.database is required." -}} +{{- else -}} +{{- .Values.secret.database | required ".Values.secret.database is required." -}} {{- end -}} {{- end -}} @@ -31,11 +91,11 @@ Create the name of the service account to use Generates the connection string for the postgresql service */}} {{- define "server.postgres-connstr" -}} -{{- $user := .Values.postgresql.auth.username -}} -{{- $pass := .Values.postgresql.auth.password | required ".Values.postgresql.auth.password is required." -}} +{{- $user := include "server.postgres-username" . -}} +{{- $pass := include "server.postgres-password" . -}} {{- $host := include "server.postgres-hostname" . -}} -{{- $port := .Values.postgresql.containerPorts.postgresql | toString -}} -{{- $db := .Values.postgresql.auth.database -}} +{{- $port := include "server.postgres-port" . -}} +{{- $db := include "server.postgres-database" . -}} {{- printf "postgresql+asyncpg://%s:%s@%s:%s/%s" $user $pass $host $port $db -}} {{- end -}} @@ -54,6 +114,8 @@ Create the name of the service account to use {{- end -}} {{- end -}} +// ----- End connection string templates ----- + {{- define "server.uiUrl" -}} {{- if .Values.server.uiConfig.prefectUiUrl -}} {{- .Values.server.uiConfig.prefectUiUrl -}} diff --git a/charts/prefect-server/values.yaml b/charts/prefect-server/values.yaml index 402c34ad..2791eaf8 100644 --- a/charts/prefect-server/values.yaml +++ b/charts/prefect-server/values.yaml @@ -293,7 +293,20 @@ ingress: ## port: ## name: http -# Postgresql configuration +# Secret configuration +secret: + # -- username for the PostgreSQL connection string + username: "" + # -- password for the PostgreSQL connection string + password: "" + # -- host for the PostgreSQL connection string + host: "" + # -- port for the PostgreSQL connection string + port: "" + # -- database for the PostgreSQL connection string + database: "" + +# PostgreSQL subchart - default overrides postgresql: # -- enable use of bitnami/postgresql subchart enabled: true From f661ebe3fe85333635f2ec18b5bd3bf91f7fd716 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 10:27:50 -0500 Subject: [PATCH 4/9] Support setting name and creation of Secret - Supports setting the name to use for the Secret - Supports controlling whether or not to create the Secret - Supports someone providing their own existing Secret --- charts/prefect-server/README.md | 2 ++ charts/prefect-server/templates/_helpers.tpl | 2 ++ charts/prefect-server/templates/secret.yaml | 2 +- charts/prefect-server/values.yaml | 6 ++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index 82b50a24..7fd1aaa4 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -145,8 +145,10 @@ No secrets are created when providing an existing secret. | postgresql.image.tag | string | `"14.3.0"` | Version tag, corresponds to tags at https://hub.docker.com/r/bitnami/postgresql/ | | postgresql.primary.initdb.user | string | `"postgres"` | specify the PostgreSQL username to execute the initdb scripts | | postgresql.primary.persistence.enabled | bool | `false` | enable PostgreSQL Primary data persistence using PVC | +| secret.create | bool | `true` | whether to create a Secret containing the PostgreSQL connection string | | secret.database | string | `""` | database for the PostgreSQL connection string | | secret.host | string | `""` | host for the PostgreSQL connection string | +| secret.name | string | `""` | name for the Secret containing the PostgreSQL connection string To provide an existing Secret, provide a name and set `create=false` | | secret.password | string | `""` | password for the PostgreSQL connection string | | secret.port | string | `""` | port for the PostgreSQL connection string | | secret.username | string | `""` | username for the PostgreSQL connection string | diff --git a/charts/prefect-server/templates/_helpers.tpl b/charts/prefect-server/templates/_helpers.tpl index 2281a879..be622a43 100644 --- a/charts/prefect-server/templates/_helpers.tpl +++ b/charts/prefect-server/templates/_helpers.tpl @@ -108,6 +108,8 @@ Create the name of the service account to use {{- define "server.postgres-string-secret-name" -}} {{- if .Values.postgresql.auth.existingSecret -}} {{- .Values.postgresql.auth.existingSecret -}} +{{- else if .Values.secret.name -}} + {{- .Values.secret.name -}} {{- else -}} {{- $name := include "common.names.fullname" . -}} {{- printf "%s-%s" $name "postgresql-connection" -}} diff --git a/charts/prefect-server/templates/secret.yaml b/charts/prefect-server/templates/secret.yaml index d5ffe4f4..a863efdd 100644 --- a/charts/prefect-server/templates/secret.yaml +++ b/charts/prefect-server/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.postgresql.auth.existingSecret }} +{{- if .Values.secret.create }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/prefect-server/values.yaml b/charts/prefect-server/values.yaml index 2791eaf8..1aa9a546 100644 --- a/charts/prefect-server/values.yaml +++ b/charts/prefect-server/values.yaml @@ -295,6 +295,12 @@ ingress: # Secret configuration secret: + # -- whether to create a Secret containing the PostgreSQL connection string + create: true + # -- name for the Secret containing the PostgreSQL connection string + # To provide an existing Secret, provide a name and set `create=false` + name: "" + # -- username for the PostgreSQL connection string username: "" # -- password for the PostgreSQL connection string From b237cb498a7698d4a3fe53f844e640da7a068fde Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 15:31:56 -0500 Subject: [PATCH 5/9] Add helm unit tests for database configuration Uses https://github.com/helm-unittest/helm-unittest to unit test the database configuration. This is a replacement for coming up with test cases locally, running `helm template`, and manually validating the output. --- README.md | 4 + charts/prefect-server/.helmignore | 2 + .../prefect-server/tests/database_test.yaml | 108 ++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 charts/prefect-server/tests/database_test.yaml diff --git a/README.md b/README.md index 8df21a16..5adec8bd 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,10 @@ Please make sure that your changes have been linted & the chart documentation ha Make sure that any new functionality is well tested! You can do this by installing the chart locally, see [above](https://github.com/PrefectHQ/prefect-helm#installing-development-versions) for how to do this. +You can also create and run test suites via [helm-unittest](https://github.com/helm-unittest/helm-unittest). +Related test files are stored under `./charts//tests/*_test.yaml`. +Refer to the `helm-unittest` repository for more information. + ### Opening a PR A helpful PR explains WHAT changed and WHY the change is important. Please take time to make your PR descriptions as helpful as possible. If you are opening a PR from a forked repository - please follow [these](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) docs to allow `prefect-helm` maintainers to push commits to your local branch. diff --git a/charts/prefect-server/.helmignore b/charts/prefect-server/.helmignore index 0e8a0eb3..a83f6664 100644 --- a/charts/prefect-server/.helmignore +++ b/charts/prefect-server/.helmignore @@ -21,3 +21,5 @@ .idea/ *.tmproj .vscode/ +# helm-unittest +tests diff --git a/charts/prefect-server/tests/database_test.yaml b/charts/prefect-server/tests/database_test.yaml new file mode 100644 index 00000000..60ed1db0 --- /dev/null +++ b/charts/prefect-server/tests/database_test.yaml @@ -0,0 +1,108 @@ +suite: Database configuration +release: + name: test + namespace: prefect + +# Anchors to reuse in the tests +envSecretPath: &envSecretPath .spec.template.spec.containers[?(@.name == "prefect-server")].env[?(@.name == "PREFECT_API_DATABASE_CONNECTION_URL")].valueFrom.secretKeyRef.name +defaultSecretName: &defaultSecretName prefect-server-postgresql-connection + +tests: + # Bundled PostgreSQL chart tests + + - it: Should produce the expected secret name and content with the defaults + asserts: + - template: secret.yaml + equal: + path: .metadata.name + value: *defaultSecretName + - template: secret.yaml + equal: + path: .data.connection-string + decodeBase64: true + value: postgresql+asyncpg://prefect:prefect-rocks@test-postgresql.prefect:5432/server + - template: deployment.yaml + equal: + path: *envSecretPath + value: *defaultSecretName + + - it: Should inject custom connection auth info + set: + postgresql: + primary: + service: + ports: + postgresql: 1234 + auth: + username: myuser + password: mypass + database: mydb + asserts: + - template: secret.yaml + equal: + path: .data.connection-string + decodeBase64: true + value: postgresql+asyncpg://myuser:mypass@test-postgresql.prefect:1234/mydb + + - it: Should configure an external secret name correctly + set: + postgresql: + auth: + existingSecret: my-pg-secret + asserts: + - template: secret.yaml + equal: + path: .metadata.name + value: my-pg-secret + - template: deployment.yaml + equal: + path: *envSecretPath + value: my-pg-secret + + + # External PostgreSQL instance tests + + - it: Should inject custom connection auth info + set: + postgresql: + enabled: false + secret: + username: foo + password: bar + host: mypghost.com + port: 1234 + database: mydb + asserts: + - template: secret.yaml + equal: + path: .data.connection-string + decodeBase64: true + value: postgresql+asyncpg://foo:bar@mypghost.com:1234/mydb + + - it: Should fail if connection info is incomplete + set: + postgresql: + enabled: false + secret: + username: foo + password: bar + # Not configured: + # host: + # port: + # database: + asserts: + - failedTemplate: + errorPattern: is required + + - it: Should configure an existing secret name correctly + set: + postgresql: + enabled: false + secret: + create: false + name: my-pg-secret + asserts: + - template: deployment.yaml + equal: + path: *envSecretPath + value: my-pg-secret From b3c14cc84332fd6f7cffebc272d36393d7eb361a Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 15:32:45 -0500 Subject: [PATCH 6/9] Add script and CI workflow for helm-unittest - Adds script to run helm-unittest locally - Adds CI workflow to run helm-unittest in GitHub Actions --- .github/workflows/helm-unittest.yaml | 20 ++++++++++++++++++++ scripts/helm_unittest.sh | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/helm-unittest.yaml create mode 100755 scripts/helm_unittest.sh diff --git a/.github/workflows/helm-unittest.yaml b/.github/workflows/helm-unittest.yaml new file mode 100644 index 00000000..e0f523b5 --- /dev/null +++ b/.github/workflows/helm-unittest.yaml @@ -0,0 +1,20 @@ +# https://github.com/marketplace/actions/helm-unit-tests + +name: Run Helm unit tests + +on: pull_request + +# Do not grant jobs any permissions by default +permissions: {} + +jobs: + unittest: + runs-on: ubuntu-latest + permissions: + # required to read from the repo + contents: read + steps: + - uses: actions/checkout@v3 + - uses: d3adb5/helm-unittest-action@v2 + with: + helm-version: v3.15.3 diff --git a/scripts/helm_unittest.sh b/scripts/helm_unittest.sh new file mode 100755 index 00000000..1ffa5d71 --- /dev/null +++ b/scripts/helm_unittest.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# This script uses https://github.com/helm-unittest/helm-unittest +# to run unit tests for our Helm Chart templates. +# +# It uses the Docker image to make it easier to run on local machines without +# having to manage the binary and its correct version. Note that if it ever +# appears in https://mise.jdx.dev/registry.html, we can add an entry in .mise.toml. +# +# Dependencies: +# - docker +# +# Usage: +# ./scripts/helm_unittest.sh + +version=${VERSION:-3.15.3-0.5.2} + +docker run \ + -it --rm \ + -v $(pwd):/apps \ + helmunittest/helm-unittest:${version} charts/prefect-server From 0f43bfc8f082caad1b2faf19a0ef74943ff345e4 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Thu, 25 Jul 2024 16:03:23 -0500 Subject: [PATCH 7/9] Update README for new external db configuration Updates the README to reflect the new configuration for external databases. --- charts/prefect-server/README.md | 45 +++++++++++++++----------- charts/prefect-server/README.md.gotmpl | 45 +++++++++++++++----------- 2 files changed, 54 insertions(+), 36 deletions(-) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index 7fd1aaa4..32d1d8e7 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -28,8 +28,14 @@ Note: If you choose to make modifications to either the `server.prefectApiUrl` o ### Handling Connection Secrets -If you are installing the chart as-is (and therefore installing PostgreSQL) - you'll need to update one of two fields: -1. `postgresql.auth.password`: a password you want to set for the prefect user +#### Using the bundled PostgreSQL chart + +By default, Bitnami's PostgreSQL Helm Chart will be deployed. This is **not intended for production use**, and is only +included to provide a functional proof of concept installation. + +In this scenario, you'll need to provide _either one_ of the following fields: + +1. `postgresql.auth.password`: a password you want to set for the prefect user (default: `prefect-rocks`) 2. `postgresql.auth.existingSecret`: name of an existing secret in your cluster with the following field: @@ -38,31 +44,34 @@ If you are installing the chart as-is (and therefore installing PostgreSQL) - yo - hostname = `-postgresql.:` - database = `postgresql.auth.database` +Two secrets are created when not providing an existing secret name: +1. `prefect-server-postgresql-connection`: used by the prefect-server deployment to connect to the postgresql database. + +2. `-postgresql-0`: defines the `postgresql.auth.username`'s password on the postgresql server to allow successful authentication from the prefect server. + +#### Using an external instance of PostgreSQL + If you want to disable the bundled PostgreSQL chart and use an external instance, provide the following configuration: ```yaml prefect-server: postgresql: - # Disable the objects from the bundled PostgreSQL chart enabled: false - auth: - # Option 1: provide the name of an existing secret following the instructions above. - existingSecret: - - # Option 2: provide the connection string details directly - username: myuser - password: mypass - database: mydb + secret: + # Option 1: provide the name of an existing secret following the instructions above. + create: false + name: + + # Option 2: provide the connection string details directly + create: true + username: myuser + password: mypass + host: myhost.com + port: 1234 + database: mydb ``` -Two secrets are created when not providing an existing secret name: -1. `prefect-server-postgresql-connection`: used by the prefect-server deployment to connect to the postgresql database. - -2. `-postgresql-0`: defines the `postgresql.auth.username`'s password on the postgresql server to allow successful authentication from the prefect server. - -No secrets are created when providing an existing secret. - ### Connecting with SSL configured 1. Mount the relevant certificate to `/home/prefect/.postgresql` so that it can be found by `asyncpg`. This is the default location postgresql expects per their [documentation](https://www.postgresql.org/docs/current/libpq-ssl.html). diff --git a/charts/prefect-server/README.md.gotmpl b/charts/prefect-server/README.md.gotmpl index 34e835de..18622fe8 100644 --- a/charts/prefect-server/README.md.gotmpl +++ b/charts/prefect-server/README.md.gotmpl @@ -27,8 +27,14 @@ Note: If you choose to make modifications to either the `server.prefectApiUrl` o ### Handling Connection Secrets -If you are installing the chart as-is (and therefore installing PostgreSQL) - you'll need to update one of two fields: -1. `postgresql.auth.password`: a password you want to set for the prefect user +#### Using the bundled PostgreSQL chart + +By default, Bitnami's PostgreSQL Helm Chart will be deployed. This is **not intended for production use**, and is only +included to provide a functional proof of concept installation. + +In this scenario, you'll need to provide _either one_ of the following fields: + +1. `postgresql.auth.password`: a password you want to set for the prefect user (default: `prefect-rocks`) 2. `postgresql.auth.existingSecret`: name of an existing secret in your cluster with the following field: @@ -37,31 +43,34 @@ If you are installing the chart as-is (and therefore installing PostgreSQL) - yo - hostname = `-postgresql.:` - database = `postgresql.auth.database` +Two secrets are created when not providing an existing secret name: +1. `prefect-server-postgresql-connection`: used by the prefect-server deployment to connect to the postgresql database. + +2. `-postgresql-0`: defines the `postgresql.auth.username`'s password on the postgresql server to allow successful authentication from the prefect server. + +#### Using an external instance of PostgreSQL + If you want to disable the bundled PostgreSQL chart and use an external instance, provide the following configuration: ```yaml prefect-server: postgresql: - # Disable the objects from the bundled PostgreSQL chart enabled: false - auth: - # Option 1: provide the name of an existing secret following the instructions above. - existingSecret: - - # Option 2: provide the connection string details directly - username: myuser - password: mypass - database: mydb + secret: + # Option 1: provide the name of an existing secret following the instructions above. + create: false + name: + + # Option 2: provide the connection string details directly + create: true + username: myuser + password: mypass + host: myhost.com + port: 1234 + database: mydb ``` -Two secrets are created when not providing an existing secret name: -1. `prefect-server-postgresql-connection`: used by the prefect-server deployment to connect to the postgresql database. - -2. `-postgresql-0`: defines the `postgresql.auth.username`'s password on the postgresql server to allow successful authentication from the prefect server. - -No secrets are created when providing an existing secret. - ### Connecting with SSL configured 1. Mount the relevant certificate to `/home/prefect/.postgresql` so that it can be found by `asyncpg`. This is the default location postgresql expects per their [documentation](https://www.postgresql.org/docs/current/libpq-ssl.html). From cc235e80050b6a04ce90a5015a46467a93186f24 Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Mon, 12 Aug 2024 16:41:24 -0500 Subject: [PATCH 8/9] Add helm/chart-testing to mise Installs the helm/chart-testing binary at its latest version. Needed for the tests defined under `charts//tests/test-*.yaml`. --- .mise.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.mise.toml b/.mise.toml index 394e475c..141de490 100644 --- a/.mise.toml +++ b/.mise.toml @@ -2,3 +2,4 @@ helm-docs = '1.13.1' pre-commit = '3.7.1' helm = '3.15' +helm-ct = '3.11.0' From d368e51e3fcccd1be44e8debb20fdc558ec3099f Mon Sep 17 00:00:00 2001 From: Mitchell Nielsen Date: Mon, 12 Aug 2024 16:42:41 -0500 Subject: [PATCH 9/9] Mention how to run the `helm-unittest` helper --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 5adec8bd..711888fb 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,15 @@ You can also create and run test suites via [helm-unittest](https://github.com/h Related test files are stored under `./charts//tests/*_test.yaml`. Refer to the `helm-unittest` repository for more information. +The following helper script will run the tests via the `helm-unittest` Docker image in case you don't have the binary installed locally: + +```shell +./scripts/helm_unittest.sh +``` + +When `helm-unittest` is available via the [`mise` registry](https://mise.jdx.dev/registry.html), we'll add it to `.mise.toml` +for easy local installation. + ### Opening a PR A helpful PR explains WHAT changed and WHY the change is important. Please take time to make your PR descriptions as helpful as possible. If you are opening a PR from a forked repository - please follow [these](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork) docs to allow `prefect-helm` maintainers to push commits to your local branch.