chore: add logging context around replicaset updates (#3326)

* chore: add more logging Signed-off-by: Zach Aller <[email protected]> * chore: remove accidental files Signed-off-by: Zach Aller <[email protected]> --------- Signed-off-by: Zach Aller <[email protected]>
argoproj · Jan 25, 2024 · 4dc0a4e · 4dc0a4e
1 parent a9f6216
commit 4dc0a4e
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 10 deletions.
diff --git a/rollout/bluegreen.go b/rollout/bluegreen.go
@@ -1,6 +1,7 @@
 package rollout
 
 import (
+ "fmt"
  "math"
  "sort"
 
@@ -22,7 +23,7 @@ func (c *rolloutContext) rolloutBlueGreen() error {
  }
  c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true)
  if err != nil {
- return err
+ return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutBlueGreen create true: %w", err)
  }
 
  // This must happen right after the new replicaset is created
@@ -82,6 +83,9 @@ func (c *rolloutContext) reconcileBlueGreenStableReplicaSet(activeSvc *corev1.Se
 
  c.log.Infof("Reconciling stable ReplicaSet '%s'", activeRS.Name)
  _, _, err := c.scaleReplicaSetAndRecordEvent(activeRS, defaults.GetReplicasOrDefault(c.rollout.Spec.Replicas))
+ if err != nil {
+ return fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileBlueGreenStableReplicaSet: %w", err)
+ }
  return err
 }
 
@@ -243,7 +247,7 @@ func (c *rolloutContext) scaleDownOldReplicaSetsForBlueGreen(oldRSs []*appsv1.Re
  // Scale down.
  _, _, err = c.scaleReplicaSetAndRecordEvent(targetRS, desiredReplicaCount)
  if err != nil {
- return false, err
+ return false, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in scaleDownOldReplicaSetsForBlueGreen: %w", err)
  }
  hasScaled = true
  }

diff --git a/rollout/canary.go b/rollout/canary.go
@@ -1,6 +1,7 @@
 package rollout
 
 import (
+ "fmt"
  "sort"
 
  appsv1 "k8s.io/api/apps/v1"
@@ -21,14 +22,14 @@ func (c *rolloutContext) rolloutCanary() error {
  if replicasetutil.PodTemplateOrStepsChanged(c.rollout, c.newRS) {
  c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false)
  if err != nil {
- return err
+ return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary with PodTemplateOrStepsChanged: %w", err)
  }
  return c.syncRolloutStatusCanary()
  }
 
  c.newRS, err = c.getAllReplicaSetsAndSyncRevision(true)
  if err != nil {
- return err
+ return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in rolloutCanary create true: %w", err)
  }
 
  err = c.podRestarter.Reconcile(c)
@@ -110,6 +111,9 @@ func (c *rolloutContext) reconcileCanaryStableReplicaSet() (bool, error) {
  _, desiredStableRSReplicaCount = replicasetutil.CalculateReplicaCountsForTrafficRoutedCanary(c.rollout, c.rollout.Status.Canary.Weights)
  }
  scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.stableRS, desiredStableRSReplicaCount)
+ if err != nil {
+ return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileCanaryStableReplicaSet:L %w", err)
+ }
  return scaled, err
 }
 
@@ -230,7 +234,7 @@ func (c *rolloutContext) scaleDownOldReplicaSetsForCanary(oldRSs []*appsv1.Repli
  // Scale down.
  _, _, err = c.scaleReplicaSetAndRecordEvent(targetRS, desiredReplicaCount)
  if err != nil {
- return totalScaledDown, err
+ return totalScaledDown, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in scaleDownOldReplicaSetsForCanary: %w", err)
  }
  scaleDownCount := *targetRS.Spec.Replicas - desiredReplicaCount
  maxScaleDown -= scaleDownCount

diff --git a/rollout/replicaset.go b/rollout/replicaset.go
@@ -172,6 +172,9 @@ func (c *rolloutContext) reconcileNewReplicaSet() (bool, error) {
  }
 
  scaled, _, err := c.scaleReplicaSetAndRecordEvent(c.newRS, newReplicasCount)
+ if err != nil {
+ return scaled, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in reconcileNewReplicaSet: %w", err)
+ }
  return scaled, err
 }
 
@@ -263,7 +266,7 @@ func (c *rolloutContext) cleanupUnhealthyReplicas(oldRSs []*appsv1.ReplicaSet) (
  }
  _, updatedOldRS, err := c.scaleReplicaSetAndRecordEvent(targetRS, newReplicasCount)
  if err != nil {
- return nil, totalScaledDown, err
+ return nil, totalScaledDown, fmt.Errorf("failed to scaleReplicaSetAndRecordEvent in cleanupUnhealthyReplicas: %w", err)
  }
  totalScaledDown += scaledDownCount
  oldRSs[i] = updatedOldRS

diff --git a/rollout/sync.go b/rollout/sync.go
@@ -283,7 +283,7 @@ func (c *rolloutContext) syncReplicasOnly() error {
  var err error
  c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false)
  if err != nil {
- return err
+ return fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in syncReplicasOnly: %w", err)
  }
  newStatus := c.rollout.Status.DeepCopy()
 
@@ -296,7 +296,7 @@ func (c *rolloutContext) syncReplicasOnly() error {
  if err := c.reconcileBlueGreenReplicaSets(activeSvc); err != nil {
  // If we get an error while trying to scale, the rollout will be requeued
  // so we can abort this resync
- return err
+ return fmt.Errorf("failed to reconcileBlueGreenReplicaSets in syncReplicasOnly: %w", err)
  }
  activeRS, _ := replicasetutil.GetReplicaSetByTemplateHash(c.allRSs, newStatus.BlueGreen.ActiveSelector)
  if activeRS != nil {
@@ -314,7 +314,7 @@ func (c *rolloutContext) syncReplicasOnly() error {
  if _, err := c.reconcileCanaryReplicaSets(); err != nil {
  // If we get an error while trying to scale, the rollout will be requeued
  // so we can abort this resync
- return err
+ return fmt.Errorf("failed to reconcileCanaryReplicaSets in syncReplicasOnly: %w", err)
  }
  newStatus.AvailableReplicas = replicasetutil.GetAvailableReplicaCountForReplicaSets(c.allRSs)
  newStatus.HPAReplicas = replicasetutil.GetActualReplicaCountForReplicaSets(c.allRSs)
@@ -330,7 +330,7 @@ func (c *rolloutContext) isScalingEvent() (bool, error) {
  var err error
  c.newRS, err = c.getAllReplicaSetsAndSyncRevision(false)
  if err != nil {
- return false, err
+ return false, fmt.Errorf("failed to getAllReplicaSetsAndSyncRevision in isScalingEvent: %w", err)
  }
 
  for _, rs := range controller.FilterActiveReplicaSets(c.allRSs) {
@@ -357,6 +357,9 @@ func (c *rolloutContext) scaleReplicaSetAndRecordEvent(rs *appsv1.ReplicaSet, ne
  scalingOperation = "down"
  }
  scaled, newRS, err := c.scaleReplicaSet(rs, newScale, c.rollout, scalingOperation)
+ if err != nil {
+ return scaled, newRS, fmt.Errorf("failed to scaleReplicaSet in scaleReplicaSetAndRecordEvent: %w", err)
+ }
  return scaled, newRS, err
 }