Backup implementation

* install velero via flux rather than code
* TODO: code removal due to the chart installation
* adjusted roles for the velero chart
* removed unnecessary controller values
* fix bug in providertemplates ctrl
  when ownerreferences are being updated
  but requeue is not set
* TODO: actually remove the code
* TODO: rework controller to ticker
  but watch the mgmt events and manage schedule
  instead of velero schedule

internal/controller/management_backup_controller.go

@@ -20,11 +20,9 @@ import (
 	"fmt"
 	"os"
 	"slices"
-	"strings"
 	"time"
 
 	velerov1api "github.com/zerospiel/velero/pkg/apis/velero/v1"
-	appsv1 "k8s.io/api/apps/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
@@ -95,18 +93,18 @@ func (r *ManagementBackupReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		backupInstance.Namespace = req.Namespace
 	}
 
-	if requestEqualsMgmt {
-		l.Info("Reconciling velero stack parts")
-		installRes, err := r.config.ReconcileVeleroInstallation(ctx, mgmt)
-		if err != nil {
-			l.Error(err, "velero stack installation")
-			return ctrl.Result{}, err
-		}
+	// if requestEqualsMgmt {
+	// 	l.Info("Reconciling velero stack parts")
+	// 	installRes, err := r.config.ReconcileVeleroInstallation(ctx, mgmt)
+	// 	if err != nil {
+	// 		l.Error(err, "velero stack installation")
+	// 		return ctrl.Result{}, err
+	// 	}
 
-		if !installRes.IsZero() {
-			return installRes, nil
-		}
-	}
+	// 	if !installRes.IsZero() {
+	// 		return installRes, nil
+	// 	}
+	// }
 
 	if btype == backup.TypeNone {
 		if requestEqualsMgmt {
@@ -139,14 +137,14 @@ func (r *ManagementBackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	// NOTE: without installed CRDs it is impossible to initialize informers
 	// and the uncached client is required because it this point the manager
 	// still has not started the cache yet
-	uncachedCl, err := client.New(mgr.GetConfig(), client.Options{Cache: nil})
-	if err != nil {
-		return fmt.Errorf("failed to create uncached client: %w", err)
-	}
+	// uncachedCl, err := client.New(mgr.GetConfig(), client.Options{Cache: nil})
+	// if err != nil {
+	// 	return fmt.Errorf("failed to create uncached client: %w", err)
+	// }
 
-	if err := r.config.InstallVeleroCRDs(uncachedCl); err != nil {
-		return fmt.Errorf("failed to install velero CRDs: %w", err)
-	}
+	// if err := r.config.InstallVeleroCRDs(uncachedCl); err != nil {
+	// 	return fmt.Errorf("failed to install velero CRDs: %w", err)
+	// }
 
 	getManagementNameIfEnabled := func(ctx context.Context) ctrl.Request {
 		mgmt, err := r.config.GetManagement(ctx)
@@ -181,28 +179,28 @@ func (r *ManagementBackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				DeleteFunc:  func(event.TypedDeleteEvent[client.Object]) bool { return false },
 			},
 		)).
-		Watches(&velerov1api.BackupStorageLocation{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []ctrl.Request {
-			return enqueueIfManagementEnabled(getManagementNameIfEnabled(ctx))
-		}), builder.WithPredicates(
-			predicate.Funcs{
-				GenericFunc: func(event.TypedGenericEvent[client.Object]) bool { return false },
-				DeleteFunc:  func(event.TypedDeleteEvent[client.Object]) bool { return false },
-				CreateFunc:  func(event.TypedCreateEvent[client.Object]) bool { return true },
-				UpdateFunc: func(tue event.TypedUpdateEvent[client.Object]) bool {
-					oldBSL, ok := tue.ObjectOld.(*velerov1api.BackupStorageLocation)
-					if !ok {
-						return false
-					}
-
-					newBSL, ok := tue.ObjectNew.(*velerov1api.BackupStorageLocation)
-					if !ok {
-						return false
-					}
-
-					return newBSL.Spec.Provider != oldBSL.Spec.Provider
-				},
-			},
-		)).
+		// Watches(&velerov1api.BackupStorageLocation{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []ctrl.Request {
+		// 	return enqueueIfManagementEnabled(getManagementNameIfEnabled(ctx))
+		// }), builder.WithPredicates(
+		// 	predicate.Funcs{
+		// 		GenericFunc: func(event.TypedGenericEvent[client.Object]) bool { return false },
+		// 		DeleteFunc:  func(event.TypedDeleteEvent[client.Object]) bool { return false },
+		// 		CreateFunc:  func(event.TypedCreateEvent[client.Object]) bool { return true },
+		// 		UpdateFunc: func(tue event.TypedUpdateEvent[client.Object]) bool {
+		// 			oldBSL, ok := tue.ObjectOld.(*velerov1api.BackupStorageLocation)
+		// 			if !ok {
+		// 				return false
+		// 			}
+
+		// 			newBSL, ok := tue.ObjectNew.(*velerov1api.BackupStorageLocation)
+		// 			if !ok {
+		// 				return false
+		// 			}
+
+		// 			return newBSL.Spec.Provider != oldBSL.Spec.Provider
+		// 		},
+		// 	},
+		// )).
 		Watches(&hmcv1alpha1.Management{}, handler.Funcs{
 			GenericFunc: nil,
 			DeleteFunc: func(_ context.Context, tde event.TypedDeleteEvent[client.Object], q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
@@ -235,18 +233,18 @@ func (r *ManagementBackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				q.Add(ctrl.Request{NamespacedName: client.ObjectKeyFromObject(tue.ObjectNew)})
 			},
 		}).
-		Watches(&appsv1.Deployment{}, handler.Funcs{
-			GenericFunc: nil,
-			DeleteFunc:  nil,
-			CreateFunc:  nil,
-			UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
-				if tue.ObjectNew.GetNamespace() != r.config.GetVeleroSystemNamespace() || tue.ObjectNew.GetName() != backup.VeleroName {
-					return
-				}
-
-				q.Add(getManagementNameIfEnabled(ctx))
-			},
-		}).
+		// Watches(&appsv1.Deployment{}, handler.Funcs{
+		// 	GenericFunc: nil,
+		// 	DeleteFunc:  nil,
+		// 	CreateFunc:  nil,
+		// 	UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
+		// 		if tue.ObjectNew.GetNamespace() != r.config.GetVeleroSystemNamespace() || tue.ObjectNew.GetName() != backup.VeleroName {
+		// 			return
+		// 		}
+
+		// 		q.Add(getManagementNameIfEnabled(ctx))
+		// 	},
+		// }).
 		Watches(&hmcv1alpha1.ClusterDeployment{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []ctrl.Request {
 			return enqueueIfManagementEnabled(getManagementNameIfEnabled(ctx))
 		}), builder.WithPredicates(
@@ -300,25 +298,25 @@ func parseEnvsToConfig(cl client.Client, mgr interface {
 },
 ) (*backup.Config, error) {
 	const (
-		installationReqDurationEnv = "BACKUP_CTRL_INSTALL_READINESS_REQUEUE_DURATION"
-		reqDurationEnv             = "BACKUP_CTRL_REQUEUE_DURATION"
+		// installationReqDurationEnv = "BACKUP_CTRL_INSTALL_READINESS_REQUEUE_DURATION"
+		reqDurationEnv = "BACKUP_CTRL_REQUEUE_DURATION"
 	)
-	installationRequeueAfter, err := time.ParseDuration(os.Getenv(installationReqDurationEnv))
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse env %s duration: %w", installationReqDurationEnv, err)
-	}
+	// installationRequeueAfter, err := time.ParseDuration(os.Getenv(installationReqDurationEnv))
+	// if err != nil {
+	// 	return nil, fmt.Errorf("failed to parse env %s duration: %w", installationReqDurationEnv, err)
+	// }
 
 	objectsRequeueAfter, err := time.ParseDuration(os.Getenv(reqDurationEnv))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse env %s duration: %w", reqDurationEnv, err)
 	}
 
 	return backup.NewConfig(cl, mgr.GetConfig(), mgr.GetScheme(),
-		backup.WithFeatures(strings.Split(strings.ReplaceAll(os.Getenv("BACKUP_FEATURES"), ", ", ","), ",")...),
-		backup.WithInstallationRequeueAfter(installationRequeueAfter),
+		// backup.WithFeatures(strings.Split(strings.ReplaceAll(os.Getenv("BACKUP_FEATURES"), ", ", ","), ",")...),
+		// backup.WithInstallationRequeueAfter(installationRequeueAfter),
 		backup.WithObjectsRequeueAfter(objectsRequeueAfter),
-		backup.WithVeleroImage(os.Getenv("BACKUP_BASIC_IMAGE")),
-		backup.WithVeleroSystemNamespace(os.Getenv("BACKUP_SYSTEM_NAMESPACE")),
-		backup.WithPluginImages(strings.Split(strings.ReplaceAll(os.Getenv("BACKUP_PLUGIN_IMAGES"), ", ", ","), ",")...),
+		// backup.WithVeleroImage(os.Getenv("BACKUP_BASIC_IMAGE")),
+		// backup.WithVeleroSystemNamespace(os.Getenv("BACKUP_SYSTEM_NAMESPACE")),
+		// backup.WithPluginImages(strings.Split(strings.ReplaceAll(os.Getenv("BACKUP_PLUGIN_IMAGES"), ", ", ","), ",")...),
 	), nil
 }

internal/controller/management_controller.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"slices"
 	"strings"
 
@@ -30,6 +31,7 @@ import (
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -554,6 +556,25 @@ func (r *ManagementReconciler) enableAdditionalComponents(ctx context.Context, m
 		capiOperatorValues = v
 	}
 
+	if config["velero"] != nil {
+		v, ok := config["velero"].(map[string]any)
+		if !ok {
+			return fmt.Errorf("failed to cast 'velero' (type %T) to map[string]any", config["cluster-api-operator"])
+		}
+
+		const veleroServiceAccountName = "VELERO_SERVICE_ACCOUNT_NAME"
+		saName := os.Getenv(veleroServiceAccountName)
+		if saName == "" {
+			return fmt.Errorf("the env %s is not set or is empty", veleroServiceAccountName)
+		}
+
+		if err := unstructured.SetNestedField(v, saName, "serviceAccount", "server", "name"); err != nil {
+			return fmt.Errorf("failed to set .velero.serviceAccount.server.name to %s: %w", saName, err)
+		}
+
+		config["velero"] = v
+	}
+
 	if r.Config != nil {
 		if err := certmanager.VerifyAPI(ctx, r.Config, r.SystemNamespace); err != nil {
 			return fmt.Errorf("failed to check in the cert-manager API is installed: %w", err)

internal/controller/template_controller.go

@@ -97,7 +97,7 @@ func (r *ClusterTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	if err := r.validateCompatibilityAttrs(ctx, clusterTemplate); err != nil {
 		if apierrors.IsNotFound(err) {
 			l.Info("Validation cannot be performed until Management cluster appears", "requeue in", defaultRequeueTime)
-			return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil
+			return ctrl.Result{RequeueAfter: defaultRequeueTime}, nil // generation has not changed, need explicit requeue
 		}
 
 		l.Error(err, "failed to validate compatibility attributes")
@@ -160,7 +160,7 @@ func (r *ProviderTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 	if changed {
 		l.Info("Updating OwnerReferences with associated Releases")
-		return ctrl.Result{}, r.Update(ctx, providerTemplate)
+		return ctrl.Result{Requeue: true}, r.Update(ctx, providerTemplate)
 	}
 
 	return r.ReconcileTemplate(ctx, providerTemplate)

templates/provider/hmc/Chart.lock

@@ -8,5 +8,8 @@ dependencies:
 - name: cluster-api-operator
   repository: https://kubernetes-sigs.github.io/cluster-api-operator
   version: 0.15.1
-digest: sha256:ba894e71230268164bfcd65813ac700776cc7da1603fd68522cdedd543468d97
-generated: "2024-12-30T09:28:33.584882+07:00"
+- name: velero
+  repository: https://vmware-tanzu.github.io/helm-charts
+  version: 8.2.0
+digest: sha256:cf8aa76af18925a80c248a60bbaa5c4d0be076e4ef772924338d5a5ec0ffaf3d
+generated: "2025-01-10T15:17:42.03581+01:00"

templates/provider/hmc/Chart.yaml

@@ -28,3 +28,7 @@ dependencies:
     version: 0.15.1
     repository: https://kubernetes-sigs.github.io/cluster-api-operator
     condition: cluster-api-operator.enabled
+  - name: velero
+    version: 8.2.0
+    repository: https://vmware-tanzu.github.io/helm-charts
+    condition: velero.enabled

templates/provider/hmc/templates/deployment.yaml

@@ -39,18 +39,10 @@ spec:
         env:
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
-        - name: BACKUP_BASIC_IMAGE
-          value: {{ template "backup.imageName" . }}
-        - name: BACKUP_FEATURES
-          value: {{ .Values.controller.backup.features }}
-        - name: BACKUP_SYSTEM_NAMESPACE
-          value: {{ .Values.controller.backup.namespace }}
-        - name: BACKUP_CTRL_INSTALL_READINESS_REQUEUE_DURATION
-          value: {{ .Values.controller.backup.installReadinessRequeuePeriod }}
+        - name: VELERO_SERVICE_ACCOUNT_NAME
+          value: {{ include "hmc.fullname" . }}-velero-server-sa
         - name: BACKUP_CTRL_REQUEUE_DURATION
           value: {{ .Values.controller.backup.requeuePeriod }}
-        - name: BACKUP_PLUGIN_IMAGES
-          value: {{ join "," .Values.controller.backup.veleroPluginImages | quote }}
         image: {{ .Values.image.repository }}:{{ .Values.image.tag
           | default .Chart.AppVersion }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}

templates/provider/hmc/templates/rbac/controller/rolebindings.yaml

@@ -28,3 +28,19 @@ subjects:
   - kind: ServiceAccount
     name: '{{ include "hmc.fullname" . }}-controller-manager'
     namespace: '{{ .Release.Namespace }}'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "hmc.fullname" . }}-velero-server-rolebinding
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "hmc.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: '{{ include "hmc.fullname" . }}-velero-server-role'
+subjects:
+  - kind: ServiceAccount
+    name: '{{ include "hmc.fullname" . }}-velero-server-sa'
+    namespace: '{{ .Release.Namespace }}'

templates/provider/hmc/templates/rbac/controller/roles.yaml

@@ -185,7 +185,7 @@ rules:
   - azureclusteridentities
   - vsphereclusteridentities
   verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - update # required for the managementbackups-ctrl
+  - update # required for the managementbackups
 - apiGroups:
   - config.projectsveltos.io
   resources:
@@ -217,8 +217,7 @@ rules:
   resources:
   - secrets
   verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - create # required for the managementbackups-ctrl
-  - update # required for the managementbackups-ctrl
+  - update # required for the managementbackups
 # managementbackups-ctrl
 - apiGroups:
   - hmc.mirantis.com
@@ -239,51 +238,12 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - ""
-  resources:
-  - serviceaccounts
-  - namespaces
-  verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - create
-  - update
-- apiGroups:
-  - apps
-  resources:
-  - deployments
-  verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - create
-  - delete
-  - patch
-- apiGroups:
-  - rbac.authorization.k8s.io
-  resources:
-  - clusterrolebindings
-  - clusterroles
-  - rolebindings
-  - roles
-  verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - create
-  - update
-- apiGroups:
-  - apiextensions.k8s.io
-  resources:
-  - customresourcedefinitions
-  verbs: {{ include "rbac.viewerVerbs" . | nindent 2 }}
-  - create
 - apiGroups:
   - velero.io
   resources:
   - '*'
   verbs:
   - '*'
-- apiGroups:
-  - '*'
-  resources:
-  - '*'
-  verbs:
-  - list
-  - get
 # managementbackups-ctrl
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -301,3 +261,18 @@ rules:
   verbs:
   - get
   - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "hmc.fullname" . }}-velero-server-role
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "hmc.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'