From 52877ad513253fcc033e9ca5db3d4a47d7f346f4 Mon Sep 17 00:00:00 2001 From: Ivo Gosemann Date: Tue, 5 Nov 2024 12:22:11 +0100 Subject: [PATCH] fix(plugin): set pending releases to failed until now helm releases stuck in pending were not touched with this fix a helm release stuck in pending is set to failed the controller will then rollback and retry the upgrade --- pkg/helm/helm.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index f81ef34a5..9bc923768 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -78,9 +78,15 @@ func InstallOrUpgradeHelmChartFromPlugin(ctx context.Context, local client.Clien return err } } - // Avoid upgrading a currently pending release. if releaseStatus, ok := isCanReleaseBeUpgraded(latestRelease); !ok { - return fmt.Errorf("cannot upgrade release %s/%s in status %s", latestRelease.Namespace, latestRelease.Name, releaseStatus.String()) + if releaseStatus.IsPending() { + log.FromContext(ctx).Info("unlocking pending release in state", "namespace", latestRelease.Namespace, "name", plugin.Name) + if err := unlockPendingRelease(restClientGetter, latestRelease); err != nil { + return fmt.Errorf("failed unlocking pending release %s/%s: %w", latestRelease.Namespace, latestRelease.Name, err) + } + return fmt.Errorf("cannot upgrade pending release %s/%s, set to state to '%s'", latestRelease.Namespace, latestRelease.Name, release.StatusFailed) + } + return fmt.Errorf("cannot upgrade release %s/%s in status '%s'", latestRelease.Namespace, latestRelease.Name, releaseStatus.String()) } log.FromContext(ctx).Info("upgrading release", "namespace", plugin.Spec.ReleaseNamespace, "name", plugin.Name) @@ -547,6 +553,19 @@ func getValueFromSecret(ctx context.Context, c client.Client, secretNamespace, s return string(valByte), nil } +// unlockPendingRelease sets the status of a pending release to failed. +func unlockPendingRelease(restClientGetter genericclioptions.RESTClientGetter, rls *release.Release) error { + cfg, err := newHelmAction(restClientGetter, rls.Namespace) + if err != nil { + return err + } + if status := rls.Info.Status; status.IsPending() { + rls.SetStatus(release.StatusFailed, fmt.Sprintf("Release set to failed from stuck '%s' state", status.String())) + return cfg.Releases.Update(rls) + } + return nil +} + func isCanReleaseBeUpgraded(r *release.Release) (release.Status, bool) { if r.Info == nil { return release.StatusUnknown, false