Merge pull request #1061 from weaveworks/870-retry

feat: add remediation retry

api/v1alpha2/terraform_types.go

@@ -261,6 +261,11 @@ type TerraformSpec struct {
 	// BranchPlanner configuration.
 	// +optional
 	BranchPlanner *BranchPlanner `json:"branchPlanner,omitempty"`
+
+	// Remediation specifies what the controller should do when reconciliation
+	// fails. The default is to not perform any action.
+	// +optional
+	Remediation *Remediation `json:"remediation,omitempty"`
 }
 
 type BranchPlanner struct {
@@ -271,6 +276,14 @@ type BranchPlanner struct {
 	EnablePathScope bool `json:"enablePathScope"`
 }
 
+type Remediation struct {
+	// Retries is the number of retries that should be attempted on failures
+	// before bailing. Defaults to '0', a negative integer denotes unlimited
+	// retries.
+	// +optional
+	Retries int64 `json:"retries,omitempty"`
+}
+
 type CloudSpec struct {
 	// +required
 	Organization string `json:"organization"`
@@ -386,6 +399,11 @@ type TerraformStatus struct {
 
 	// +optional
 	Lock LockStatus `json:"lock,omitempty"`
+
+	// ReconciliationFailures is the number of reconciliation
+	// failures since the last success or update.
+	// +optional
+	ReconciliationFailures int64 `json:"reconciliationFailures,omitempty"`
 }
 
 // LockStatus defines the observed state of a Terraform State Lock
@@ -511,8 +529,9 @@ const (
 
 // The potential reasons that are associated with condition types
 const (
-	ArtifactFailedReason            = "ArtifactFailed"
 	AccessDeniedReason              = "AccessDenied"
+	ArtifactFailedReason            = "ArtifactFailed"
+	RetryLimitReachedReason         = "RetryLimitReached"
 	DeletionBlockedByDependants     = "DeletionBlockedByDependantsReason"
 	DependencyNotReadyReason        = "DependencyNotReady"
 	DriftDetectedReason             = "DriftDetected"
@@ -826,6 +845,29 @@ func TerraformStateLocked(terraform Terraform, lockID, message string) Terraform
 	return terraform
 }
 
+// TerraformReachedLimit will set a new condition on the Terraform resource
+// indicating that the resource has reached its retry limit.
+func TerraformReachedLimit(terraform Terraform) Terraform {
+	newCondition := metav1.Condition{
+		Type:    meta.StalledCondition,
+		Status:  metav1.ConditionTrue,
+		Reason:  RetryLimitReachedReason,
+		Message: "Resource reached maximum number of retries.",
+	}
+	apimeta.SetStatusCondition(terraform.GetStatusConditions(), newCondition)
+
+	return terraform
+}
+
+// TerraformResetRetry will set a new condition on the Terraform resource
+// indicating that the resource retry count has been reset.
+func TerraformResetRetry(terraform Terraform) Terraform {
+	apimeta.RemoveStatusCondition(terraform.GetStatusConditions(), meta.StalledCondition)
+	terraform.resetReconciliationFailures()
+
+	return terraform
+}
+
 // HasDrift returns true if drift has been detected since the last successful apply
 func (in Terraform) HasDrift() bool {
 	for _, condition := range in.Status.Conditions {
@@ -900,6 +942,34 @@ func (in *Terraform) GetRunnerHostname(target string, clusterDomain string) stri
 	}
 }
 
+func (in *Terraform) GetRetries() int64 {
+	if in.Spec.Remediation == nil {
+		return 0
+	}
+
+	return in.Spec.Remediation.Retries
+}
+
+func (in *Terraform) GetReconciliationFailures() int64 {
+	return in.Status.ReconciliationFailures
+}
+
+func (in *Terraform) IncrementReconciliationFailures() {
+	in.Status.ReconciliationFailures++
+}
+
+func (in *Terraform) resetReconciliationFailures() {
+	in.Status.ReconciliationFailures = 0
+}
+
+func (in *Terraform) ShouldRetry() bool {
+	if in.Spec.Remediation == nil || in.Spec.Remediation.Retries < 0 {
+		return true
+	}
+
+	return in.GetReconciliationFailures() < in.Spec.Remediation.Retries
+}
+
 func (in *TerraformSpec) GetAlwaysCleanupRunnerPod() bool {
 	if in.AlwaysCleanupRunnerPod == nil {
 		return true

charts/tf-controller/templates/crds.yaml

@@ -5269,6 +5269,17 @@ spec:
                 description: RefreshBeforeApply forces refreshing of the state before
                   the apply step.
                 type: boolean
+              remediation:
+                description: Remediation holds the remediation configuration for when
+                  the reconciliation fails. The default is to not perform any action.
+                properties:
+                  retries:
+                    description: Retries is the number of retries that should be attempted
+                      on failures before bailing. Defaults to '0', a negative integer
+                      equals to unlimited retries.
+                    format: int64
+                    type: integer
+                type: object
               retryInterval:
                 description: The interval at which to retry a previously failed reconciliation.
                   The default value is 15 when not specified.
@@ -10035,6 +10046,8 @@ spec:
             - sourceRef
             type: object
           status:
+            default:
+              observedGeneration: -1
             description: TerraformStatus defines the observed state of Terraform
             properties:
               availableOutputs:
@@ -10197,6 +10210,11 @@ spec:
                   pending:
                     type: string
                 type: object
+              reconciliationFailures:
+                description: ReconciliationFailures is the counter to track the number
+                  of reconciliation failures.
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

config/crd/bases/infra.contrib.fluxcd.io_terraforms.yaml

@@ -5267,6 +5267,17 @@ spec:
                 description: RefreshBeforeApply forces refreshing of the state before
                   the apply step.
                 type: boolean
+              remediation:
+                description: Remediation specifies what the controller should do when
+                  reconciliation fails. The default is to not perform any action.
+                properties:
+                  retries:
+                    description: Retries is the number of retries that should be attempted
+                      on failures before bailing. Defaults to '0', a negative integer
+                      denotes unlimited retries.
+                    format: int64
+                    type: integer
+                type: object
               retryInterval:
                 description: The interval at which to retry a previously failed reconciliation.
                   The default value is 15 when not specified.
@@ -10206,6 +10217,11 @@ spec:
                   pending:
                     type: string
                 type: object
+              reconciliationFailures:
+                description: ReconciliationFailures is the number of reconciliation
+                  failures since the last success or update.
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

config/tilt/test/tf-dev-subject.yaml

@@ -27,6 +27,8 @@ spec:
   path: ./
   interval: 20s
   approvePlan: auto
+  remediation:
+    retries: 3
   sourceRef:
     kind: GitRepository
     name: helloworld