Merge pull request #855 from weaveworks/3127-backport-brack-the-glass

feat: backport break-the-glass to v0.14

.gitignore

@@ -42,3 +42,6 @@ example/
 
 # OS files
 **/.DS_Store
+
+Tiltfile.local
+/config/tilt/helm/dev-values-local.yaml

Dockerfile

@@ -34,8 +34,8 @@ LABEL org.opencontainers.image.source="https://github.com/weaveworks/tf-controll
 
 RUN apk update
 
-RUN apk add --no-cache libcrypto3=3.1.1-r1 && \
-    apk add --no-cache libssl3=3.1.1-r1 && \
+RUN apk add --no-cache libcrypto3=3.1.2-r0 && \
+    apk add --no-cache libssl3=3.1.2-r0 && \
     apk add --no-cache ca-certificates tini git openssh-client gnupg && \
     apk add --no-cache libretls && \
     apk add --no-cache busybox

Tiltfile

@@ -0,0 +1,65 @@
+load('ext://restart_process', 'docker_build_with_restart')
+load('ext://helm_remote', 'helm_remote')
+load('ext://secret', 'secret_from_dict')
+load('ext://namespace', 'namespace_create', 'namespace_inject')
+
+namespace       = "flux-system"
+tfNamespace     = "terraform"
+buildSHA        = str(local('git rev-parse --short HEAD')).rstrip('\n')
+buildVersionRef = str(local('git rev-list --tags --max-count=1')).rstrip('\n')
+buildVersion    = str(local("git describe --tags ${buildVersionRef}")).rstrip('\n')
+
+if os.path.exists('Tiltfile.local'):
+   include('Tiltfile.local')
+
+namespace_create(tfNamespace)
+
+# Download chart deps
+local_resource("helm-dep-update", "helm dep update charts/tf-controller", trigger_mode=TRIGGER_MODE_MANUAL, auto_init=True)
+
+# Define resources
+k8s_resource('chart-tf-controller',
+  labels=["deployments"],
+  new_name='controller')
+
+helm_values = ['config/tilt/helm/dev-values.yaml']
+if os.path.exists('config/tilt/helm/dev-values-local.yaml'):
+   helm_values.append('config/tilt/helm/dev-values-local.yaml')
+
+k8s_yaml(helm(
+   "charts/tf-controller",
+   namespace=namespace,
+   values=helm_values,
+))
+
+# Add Example
+k8s_yaml("./config/tilt/test/tf-dev-subject.yaml")
+k8s_resource(
+  objects=['helloworld:GitRepository:terraform','helloworld-tf:Secret:terraform','helloworld-tf:Terraform:terraform'],
+  extra_pod_selectors={'instance': 'helloworld-tf'},
+  new_name="helloworld-tf",
+  pod_readiness='ignore',
+  labels=["resources"],
+)
+
+# Images
+docker_build(
+  "ghcr.io/weaveworks/tf-controller",
+  "",
+  dockerfile="Dockerfile",
+  build_args={
+    'BUILD_SHA': buildSHA,
+    'BUILD_VERSION': buildVersion,
+  })
+
+# There are no resources using this image when tilt starts, but we still need
+# this image.
+update_settings(suppress_unused_image_warnings=["ghcr.io/weaveworks/tf-runner"])
+docker_build(
+  'ghcr.io/weaveworks/tf-runner',
+  '',
+  dockerfile='runner.Dockerfile',
+  build_args={
+    'BUILD_SHA': buildSHA,
+    'BUILD_VERSION': buildVersion,
+  })

api/v1alpha1/terraform_types.go

@@ -37,11 +37,12 @@ const (
 	CACertSecretName = "tf-controller.tls"
 	// RunnerTLSSecretName is the name of the secret containing a TLS cert that will be written to
 	// the namespace in which a terraform runner is created
-	RunnerTLSSecretName   = "terraform-runner.tls"
-	RunnerLabel           = "infra.contrib.fluxcd.io/terraform"
-	GitRepositoryIndexKey = ".metadata.gitRepository"
-	BucketIndexKey        = ".metadata.bucket"
-	OCIRepositoryIndexKey = ".metadata.ociRepository"
+	RunnerTLSSecretName     = "terraform-runner.tls"
+	RunnerLabel             = "infra.contrib.fluxcd.io/terraform"
+	GitRepositoryIndexKey   = ".metadata.gitRepository"
+	BucketIndexKey          = ".metadata.bucket"
+	OCIRepositoryIndexKey   = ".metadata.ociRepository"
+	BreakTheGlassAnnotation = "break-the-glass.tf-controller/requestedAt"
 )
 
 type ReadInputsFromSecretSpec struct {
@@ -236,6 +237,11 @@ type TerraformSpec struct {
 	// Enterprise is the enterprise configuration placeholder.
 	// +optional
 	Enterprise *apiextensionsv1.JSON `json:"enterprise,omitempty"`
+
+	// BreakTheGlass specifies if the reconciliation should stop
+	// and allow interactive shell in case of emergency.
+	// +optional
+	BreakTheGlass bool `json:"breakTheGlass,omitempty"`
 }
 
 type CloudSpec struct {

api/v1alpha2/terraform_types.go

@@ -37,11 +37,12 @@ const (
 	CACertSecretName = "tf-controller.tls"
 	// RunnerTLSSecretName is the name of the secret containing a TLS cert that will be written to
 	// the namespace in which a terraform runner is created
-	RunnerTLSSecretName   = "terraform-runner.tls"
-	RunnerLabel           = "infra.contrib.fluxcd.io/terraform"
-	GitRepositoryIndexKey = ".metadata.gitRepository"
-	BucketIndexKey        = ".metadata.bucket"
-	OCIRepositoryIndexKey = ".metadata.ociRepository"
+	RunnerTLSSecretName     = "terraform-runner.tls"
+	RunnerLabel             = "infra.contrib.fluxcd.io/terraform"
+	GitRepositoryIndexKey   = ".metadata.gitRepository"
+	BucketIndexKey          = ".metadata.bucket"
+	OCIRepositoryIndexKey   = ".metadata.ociRepository"
+	BreakTheGlassAnnotation = "break-the-glass.tf-controller/requestedAt"
 )
 
 type ReadInputsFromSecretSpec struct {
@@ -236,6 +237,11 @@ type TerraformSpec struct {
 	// Enterprise is the enterprise configuration placeholder.
 	// +optional
 	Enterprise *apiextensionsv1.JSON `json:"enterprise,omitempty"`
+
+	// BreakTheGlass specifies if the reconciliation should stop
+	// and allow interactive shell in case of emergency.
+	// +optional
+	BreakTheGlass bool `json:"breakTheGlass,omitempty"`
 }
 
 type CloudSpec struct {

charts/tf-controller/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: tf-controller
 description: The Helm chart for Weave GitOps Terraform Controller
 type: application
-version: 0.11.5-lts
-appVersion: "v0.14.5"
+version: 0.11.6-lts
+appVersion: "v0.14.6"

charts/tf-controller/README.md

@@ -1,6 +1,6 @@
 # Weave GitOps Terraform Controller
 
-![Version: 0.11.4-lts](https://img.shields.io/badge/Version-0.11.4--lts-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.4](https://img.shields.io/badge/AppVersion-v0.14.4-informational?style=flat-square)
+![Version: 0.11.6-lts](https://img.shields.io/badge/Version-0.11.6--lts-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.14.6](https://img.shields.io/badge/AppVersion-v0.14.6-informational?style=flat-square)
 
 The Helm chart for Weave GitOps Terraform Controller
 
@@ -27,6 +27,7 @@ __Note__: If you need to use the `imagePullSecrets` it would be best to set `ser
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity properties for the TF-Controller deployment |
+| allowBreakTheGlass | bool | `false` | Argument for `--allow-break-the-glass` (Controller).  AllowBreakTheGlass allows the controller to break the glass and modify Terraform states when the sync loop is broken. |
 | awsPackage.install | bool | `true` |  |
 | awsPackage.repository | string | `"ghcr.io/tf-controller/aws-primitive-modules"` |  |
 | awsPackage.tag | string | `"v4.38.0-v1alpha11"` |  |
@@ -69,7 +70,7 @@ __Note__: If you need to use the `imagePullSecrets` it would be best to set `ser
 | rbac.create | bool | `true` | If `true`, create and use RBAC resources |
 | replicaCount | int | `1` | Number of TF-Controller pods to deploy |
 | resources | object | `{"limits":{"cpu":"1000m","memory":"1Gi"},"requests":{"cpu":"200m","memory":"64Mi"}}` | Resource limits and requests |
-| runner | object | `{"creationTimeout":"5m0s","grpc":{"maxMessageSize":4},"image":{"repository":"ghcr.io/weaveworks/tf-runner","tag":"v0.14.4"},"serviceAccount":{"allowedNamespaces":[],"annotations":{},"create":true,"name":""}}` | Runner-specific configurations |
+| runner | object | `{"creationTimeout":"5m0s","grpc":{"maxMessageSize":4},"image":{"repository":"ghcr.io/weaveworks/tf-runner","tag":"v0.14.6"},"serviceAccount":{"allowedNamespaces":[],"annotations":{},"create":true,"name":""}}` | Runner-specific configurations |
 | runner.creationTimeout | string | `"5m0s"` | Timeout for runner-creation (Controller) |
 | runner.grpc.maxMessageSize | int | `4` | Maximum GRPC message size (Controller) |
 | runner.image.repository | string | `"ghcr.io/weaveworks/tf-runner"` | Runner image repository |

charts/tf-controller/templates/crds.yaml

@@ -111,6 +111,10 @@ spec:
                   - name
                   type: object
                 type: array
+              breakTheGlass:
+                description: BreakTheGlass specifies if the reconciliation should
+                  stop and allow interactive shell in case of emergency.
+                type: boolean
               cliConfigSecretRef:
                 description: SecretReference represents a Secret Reference. It has
                   enough information to retrieve secret in any namespace
@@ -296,8 +300,7 @@ spec:
                 type: boolean
               retryInterval:
                 description: The interval at which to retry a previously failed reconciliation.
-                  When not specified, the controller uses the TerraformSpec.Interval
-                  value to retry failures.
+                  The default value is 15 when not specified.
                 type: string
               runnerPodTemplate:
                 properties:
@@ -5025,6 +5028,10 @@ spec:
                   - name
                   type: object
                 type: array
+              breakTheGlass:
+                description: BreakTheGlass specifies if the reconciliation should
+                  stop and allow interactive shell in case of emergency.
+                type: boolean
               cliConfigSecretRef:
                 description: SecretReference represents a Secret Reference. It has
                   enough information to retrieve secret in any namespace

charts/tf-controller/templates/deployment.yaml

@@ -42,6 +42,7 @@ spec:
         - --events-addr={{ .Values.eventsAddress }}
         - --kube-api-qps={{ .Values.kubeAPIQPS }}
         - --kube-api-burst={{ .Values.kubeAPIBurst }}
+        - --allow-break-the-glass={{ .Values.allowBreakTheGlass }}
         command:
         - /sbin/tini
         - --

charts/tf-controller/values.yaml

@@ -27,7 +27,7 @@ image:
   pullPolicy: IfNotPresent
   # -- Overrides the image tag whose default is the chart appVersion.
   # @default -- `.Chart.AppVersion`
-  tag: "v0.14.5"
+  tag: "v0.14.6"
 # -- Controller image pull secret
 imagePullSecrets: []
 # -- Resource limits and requests
@@ -97,6 +97,9 @@ kubeAPIQPS: 50
 # -- Argument for `--kube-api-burst` (Controller).
 #  Burst indicates the maximum burst queries-per-second of requests sent to the Kubernetes API, defaults to 100.
 kubeAPIBurst: 100
+# -- Argument for `--allow-break-the-glass` (Controller).
+#  AllowBreakTheGlass allows the controller to break the glass and modify Terraform states when the sync loop is broken.
+allowBreakTheGlass: false
 awsPackage:
   install: true
   tag: v4.38.0-v1alpha11
@@ -108,7 +111,7 @@ runner:
     repository: ghcr.io/weaveworks/tf-runner
     # -- Runner image tag
     # @default -- `.Chart.AppVersion`
-    tag: "v0.14.5"
+    tag: "v0.14.6"
   grpc:
     # -- Maximum GRPC message size (Controller)
     maxMessageSize: 4

cmd/manager/main.go

@@ -87,6 +87,7 @@ func main() {
 		runnerGRPCPort           int
 		runnerCreationTimeout    time.Duration
 		runnerGRPCMaxMessageSize int
+		allowBreakTheGlass       bool
 	)
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
@@ -106,6 +107,7 @@ func main() {
 	flag.IntVar(&runnerGRPCPort, "runner-grpc-port", 30000, "The port which will be exposed on the runner pod for gRPC connections.")
 	flag.DurationVar(&runnerCreationTimeout, "runner-creation-timeout", 120*time.Second, "Timeout for creating a runner pod.")
 	flag.IntVar(&runnerGRPCMaxMessageSize, "runner-grpc-max-message-size", 4, "The maximum message size for gRPC connections in MiB.")
+	flag.BoolVar(&allowBreakTheGlass, "allow-break-the-glass", false, "Allow break the glass mode.")
 
 	clientOptions.BindFlags(flag.CommandLine)
 	logOptions.BindFlags(flag.CommandLine)
@@ -190,6 +192,7 @@ func main() {
 		RunnerGRPCPort:           runnerGRPCPort,
 		RunnerCreationTimeout:    runnerCreationTimeout,
 		RunnerGRPCMaxMessageSize: runnerGRPCMaxMessageSize,
+		AllowBreakTheGlass:       allowBreakTheGlass,
 	}
 
 	if err = reconciler.SetupWithManager(mgr, concurrent, httpRetry); err != nil {

cmd/tfctl/main.go

@@ -33,7 +33,7 @@ func newRootCommand() *cobra.Command {
 
 	rootCmd := &cobra.Command{
 		Use:           "tfctl",
-		SilenceErrors: false,
+		SilenceErrors: true,
 		SilenceUsage:  true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			k8sConfig, err := kubeconfigArgs.ToRESTConfig()
@@ -68,6 +68,8 @@ func newRootCommand() *cobra.Command {
 	rootCmd.AddCommand(buildGetGroup(app))
 	rootCmd.AddCommand(buildShowGroup(app))
 
+	rootCmd.AddCommand(buildBreakTheGlassCmd(app))
+
 	return rootCmd
 }
 
@@ -350,6 +352,20 @@ func buildReplanCmd(app *tfctl.CLI) *cobra.Command {
 	return replan
 }
 
+func buildBreakTheGlassCmd(app *tfctl.CLI) *cobra.Command {
+	breakTheGlass := &cobra.Command{
+		Use:     "break-glass",
+		Aliases: []string{"break-the-glass", "bg", "btg"},
+		Short:   "Break the glass",
+		Args:    cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return app.BreakTheGlass(os.Stdout, args[0])
+		},
+	}
+
+	return breakTheGlass
+}
+
 func configureDefaultNamespace() {
 	*kubeconfigArgs.Namespace = defaultNamespace
 	fromEnv := os.Getenv("FLUX_SYSTEM_NAMESPACE")

config/crd/bases/infra.contrib.fluxcd.io_terraforms.yaml

@@ -109,6 +109,10 @@ spec:
                   - name
                   type: object
                 type: array
+              breakTheGlass:
+                description: BreakTheGlass specifies if the reconciliation should
+                  stop and allow interactive shell in case of emergency.
+                type: boolean
               cliConfigSecretRef:
                 description: SecretReference represents a Secret Reference. It has
                   enough information to retrieve secret in any namespace
@@ -5022,6 +5026,10 @@ spec:
                   - name
                   type: object
                 type: array
+              breakTheGlass:
+                description: BreakTheGlass specifies if the reconciliation should
+                  stop and allow interactive shell in case of emergency.
+                type: boolean
               cliConfigSecretRef:
                 description: SecretReference represents a Secret Reference. It has
                   enough information to retrieve secret in any namespace

config/manager/kustomization.yaml

@@ -8,4 +8,4 @@ generatorOptions:
 images:
   - name: weaveworks/tf-controller
     newName: ghcr.io/weaveworks/tf-controller
-    newTag: v0.14.5
+    newTag: v0.14.6

config/tilt/configMap.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: branch-planner
+data:
+  secretName: bbp-token
+  resources: |-
+    - namespace: terraform

config/tilt/helm/dev-values.yaml

@@ -0,0 +1,9 @@
+runner:
+  image:
+    repository: ghcr.io/weaveworks/tf-runner
+  serviceAccount:
+    allowedNamespaces:
+      - flux-system
+      - terraform
+
+allowBreakTheGlass: true