pv k8s sequence

Signed-off-by: Raghavendra Talur <[email protected]>

internal/controller/status.go

@@ -30,6 +30,10 @@ const (
 	// after this condition is true.
 	VRGConditionTypeClusterDataReady = "ClusterDataReady"
 
+	// K8s objects are ready.  This condition is used to indicate that all the
+	// K8s objects required for the app to be active in the cluster are ready.
+	VRGConditionTypeK8sObjectsReady = "K8sObjectsReady"
+
 	// PV cluster data is protected.  This condition indicates whether an app,
 	// which is active in a cluster, has all its PV related cluster data
 	// protected from a disaster by uploading it to the required S3 store(s).
@@ -58,6 +62,7 @@ const (
 	VRGConditionReasonDataProtected               = "DataProtected"
 	VRGConditionReasonProgressing                 = "Progressing"
 	VRGConditionReasonClusterDataRestored         = "Restored"
+	VRGConditionReasonK8sObjectsRestored          = "K8sObjectsRestored"
 	VRGConditionReasonError                       = "Error"
 	VRGConditionReasonErrorUnknown                = "UnknownError"
 	VRGConditionReasonUploading                   = "Uploading"
@@ -116,6 +121,14 @@ func setVRGInitialCondition(conditions *[]metav1.Condition, observedGeneration i
 		LastTransitionTime: time,
 		Message:            message,
 	})
+	setStatusConditionIfNotFound(conditions, metav1.Condition{
+		Type:               VRGConditionTypeK8sObjectsReady,
+		Reason:             VRGConditionReasonInitializing,
+		ObservedGeneration: observedGeneration,
+		Status:             metav1.ConditionUnknown,
+		LastTransitionTime: time,
+		Message:            message,
+	})
 }
 
 // sets conditions when VRG as Secondary is replicating the data with Primary.
@@ -371,6 +384,28 @@ func newVRGClusterDataUnprotectedCondition(observedGeneration int64, reason, mes
 	}
 }
 
+// sets conditions when K8s objects are restored
+func setVRGK8sObjectsReadyCondition(conditions *[]metav1.Condition, observedGeneration int64, message string) {
+	setStatusCondition(conditions, metav1.Condition{
+		Type:               VRGConditionTypeK8sObjectsReady,
+		Reason:             VRGConditionReasonK8sObjectsRestored,
+		ObservedGeneration: observedGeneration,
+		Status:             metav1.ConditionTrue,
+		Message:            message,
+	})
+}
+
+// sets conditions when K8s objects failed to restore
+func setVRGK8sObjectsErrorCondition(conditions *[]metav1.Condition, observedGeneration int64, message string) {
+	setStatusCondition(conditions, metav1.Condition{
+		Type:               VRGConditionTypeK8sObjectsReady,
+		Reason:             VRGConditionReasonError,
+		ObservedGeneration: observedGeneration,
+		Status:             metav1.ConditionFalse,
+		Message:            message,
+	})
+}
+
 func setStatusConditionIfNotFound(existingConditions *[]metav1.Condition, newCondition metav1.Condition) {
 	if existingConditions == nil {
 		existingConditions = &[]metav1.Condition{}

internal/controller/volumereplicationgroup_controller.go

@@ -640,10 +640,11 @@ func (v *VRGInstance) clusterDataRestore(result *ctrl.Result) (int, error) {
 		return numRestoredForVS + numRestoredForVR, fmt.Errorf("failed to restore PV/PVC for VolRep (%w)", err)
 	}
 
-	// Only after both succeed, we mark ClusterDataReady as true
-	msg := "Restored PVs and PVCs"
+	var msg string
 	if numRestoredForVS+numRestoredForVR == 0 {
 		msg = "Nothing to restore"
+	} else {
+		msg = fmt.Sprintf("Restored %d volsync PVs/PVCs and %d volrep PVs/PVCs", numRestoredForVS, numRestoredForVR)
 	}
 
 	setVRGClusterDataReadyCondition(&v.instance.Status.Conditions, v.instance.Generation, msg)
@@ -1183,6 +1184,18 @@ func (v *VRGInstance) processAsPrimary() ctrl.Result {
 
 	v.reconcileAsPrimary()
 
+	// restore kube objects
+	if v.shouldRestoreK8sObjects() {
+		err := v.kubeObjectsRecover(&v.result)
+		if err != nil {
+			v.log.Info("Kube objects restore failed")
+			v.errorConditionLogAndSet(err, "Failed to restore k8s objects", setVRGK8sObjectsErrorCondition)
+
+			return v.updateVRGStatus(v.result)
+		}
+		v.log.Info("Kube objects restored", "count", objectsRestored)
+	}
+
 	// If requeue is false, then VRG was successfully processed as primary.
 	// Hence the event to be generated is Success of type normal.
 	// Expectation is that, if something failed and requeue is true, then
@@ -1224,6 +1237,28 @@ func (v *VRGInstance) shouldRestoreClusterData() bool {
 	return true
 }
 
+func (v *VRGInstance) shouldRestoreK8sObjects() bool {
+	k8sObjectsRestored := findCondition(v.instance.Status.Conditions, VRGConditionTypeK8sObjectsReady)
+	if k8sObjectsRestored != nil {
+		v.log.Info("K8sObjectsReady condition",
+			"status", k8sObjectsRestored.Status,
+			"reason", k8sObjectsRestored.Reason,
+			"message", k8sObjectsRestored.Message,
+			"observedGeneration", k8sObjectsRestored.ObservedGeneration,
+			"generation", v.instance.Generation,
+		)
+
+		if k8sObjectsRestored.Status == metav1.ConditionTrue &&
+			k8sObjectsRestored.ObservedGeneration == v.instance.Generation {
+			v.log.Info("VRG's K8sObjectsReady condition found. All k8s objects must have already been restored")
+
+			return false
+		}
+	}
+
+	return true
+}
+
 func (v *VRGInstance) reconcileAsPrimary() {
 	var finalSyncPrepared struct {
 		volSync bool

internal/controller/vrg_kubeobjects.go

@@ -537,7 +537,15 @@ func (v *VRGInstance) kubeObjectsRecover(result *ctrl.Result, s3ProfileName stri
 	v.instance.Status.KubeObjectProtection.CaptureToRecoverFrom = captureToRecoverFromIdentifier
 	log := v.log.WithValues("number", captureToRecoverFromIdentifier.Number, "profile", s3ProfileName)
 
-	return v.kubeObjectsRecoveryStartOrResume(result, s3ProfileName, captureToRecoverFromIdentifier, log)
+	err = v.kubeObjectsRecoveryStartOrResume(result, s3ProfileName, captureToRecoverFromIdentifier, log)
+	if err != nil {
+		return fmt.Errorf("kube objects recovery error: %v", err)
+	}
+
+	setVRGK8sObjectsReadyCondition(&v.instance.Status.Conditions,
+		v.instance.Generation, "Kube objects restored")
+
+	return nil
 }
 
 func (v *VRGInstance) findS3StoreAccessor(s3ProfileName string) (s3StoreAccessor, error) {

internal/controller/vrg_volrep.go

@@ -2044,7 +2044,7 @@ func (v *VRGInstance) restorePVsAndPVCsFromS3(result *ctrl.Result) (int, error)
 
 		v.log.Info(fmt.Sprintf("Restored %d PVs and %d PVCs using profile %s", pvCount, pvcCount, s3ProfileName))
 
-		return pvCount + pvcCount, v.kubeObjectsRecover(result, s3ProfileName)
+		return pvCount + pvcCount, nil
 	}
 
 	if NoS3 {