Skip to content

Commit

Permalink
update func canary step
Browse files Browse the repository at this point in the history 
Signed-off-by: Bryce-Huang <[email protected]>
  • Loading branch information
@Bryce-huang
Bryce-huang committed Sep 25, 2023
1 parent 8882e05 commit 2eaa968
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 50 deletions.
16 changes: 12 additions & 4 deletions config/bundle.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39600,10 +39600,18 @@ subjects:
---
apiVersion: v1
data:
controller_manager_config.yaml: "apiVersion: controller-runtime.sigs.k8s.io/v1alpha1\r\nkind:
ControllerManagerConfig\r\nhealth:\r\n healthProbeBindAddress: :8081\r\nmetrics:\r\n
\ bindAddress: 127.0.0.1:8080\r\nwebhook:\r\n port: 9443\r\nleaderElection:\r\n
\ leaderElect: true\r\n resourceName: 79f0111e.openfunction.io\r\n"
controller_manager_config.yaml: |
apiVersion: controller-runtime.sigs.k8s.io/v1alpha1
kind: ControllerManagerConfig
health:
healthProbeBindAddress: :8081
metrics:
bindAddress: 127.0.0.1:8080
webhook:
port: 9443
leaderElection:
leaderElect: true
resourceName: 79f0111e.openfunction.io
kind: ConfigMap
metadata:
name: openfunction-manager-config
Expand Down
102 changes: 56 additions & 46 deletions controllers/core/function_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,13 @@ func (r *FunctionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
if err := r.createOrUpdateHTTPRoute(&fn); err != nil {
return ctrl.Result{}, err
}

// for canary rollback
if err := r.cleanServing(&fn); err != nil {
log.Error(err, "Failed to clean Serving")
return ctrl.Result{}, err
}

if recheckTime != nil {
return ctrl.Result{RequeueAfter: time.Until(*recheckTime)}, nil
}
Expand All @@ -160,23 +167,12 @@ func (r *FunctionReconciler) updateCanaryRelease(fn *openfunction.Function) (*ti
log := r.Log.WithName("UpdateCanaryRelease")

if !hasCanaryReleasePlan(fn) {
if fn.Status.RolloutStatus == nil || fn.Status.RolloutStatus.Canary == nil {
return nil, nil
}
fn.Status.RolloutStatus.Canary.Serving = fn.Status.Serving.DeepCopy()
fn.Status.RolloutStatus.Canary.Revision = fn.Status.Revision.DeepCopy()
// no plan no status
fn.Status.RolloutStatus.Canary.CanaryStepStatus = nil
if err := r.Status().Update(r.ctx, fn); err != nil {
log.Error(err, "Failed to update function canary status")
return nil, err
}
// sure clean old serving
if err := r.cleanServing(fn); err != nil {
log.Error(err, "Failed to clean Serving")
return nil, err
}
return nil, nil
}

if fn.Status.RolloutStatus.Canary.CanaryStepStatus == nil {
Expand All @@ -192,13 +188,12 @@ func (r *FunctionReconciler) updateCanaryRelease(fn *openfunction.Function) (*ti
}
return nil, nil
}

status := fn.Status.RolloutStatus.Canary.CanaryStepStatus

if status.Phase != openfunction.CanaryPhaseProgressing {
if !inCanaryProgress(fn) {
// not in canary release progress
return nil, nil
}
status := fn.Status.RolloutStatus.Canary.CanaryStepStatus

var recheckTime *time.Time
steps := fn.Spec.RolloutStrategy.Canary.Steps
currentStep := steps[status.CurrentStepIndex]
Expand Down Expand Up @@ -252,6 +247,24 @@ func hasCanaryReleasePlan(fn *openfunction.Function) bool {

return true
}
func inCanaryProgress(fn *openfunction.Function) bool {
if !hasCanaryReleasePlan(fn) {
return false
}
if fn.Status.RolloutStatus == nil {
return false
}
if fn.Status.RolloutStatus.Canary == nil {
return false
}
if fn.Status.RolloutStatus.Canary.Serving == nil {
return false
}
if fn.Status.RolloutStatus.Canary.CanaryStepStatus == nil {
return false
}
return fn.Status.RolloutStatus.Canary.CanaryStepStatus.Phase == openfunction.CanaryPhaseProgressing
}

func (r *FunctionReconciler) createBuilder(fn *openfunction.Function) error {
log := r.Log.WithName("CreateBuilder").
Expand Down Expand Up @@ -512,21 +525,22 @@ func (r *FunctionReconciler) createServing(fn *openfunction.Function) error {
if err := r.updateFuncWithServingStatus(fn, fn.Status.Serving); err != nil {
return err
}

if fn.Status.Serving.ResourceHash == fn.Status.RolloutStatus.Canary.Serving.ResourceHash {
fn.Status.RolloutStatus.Canary.Serving = fn.Status.Serving.DeepCopy()
fn.Status.RolloutStatus.Canary.Revision = fn.Status.Revision.DeepCopy()
err := r.Status().Update(r.ctx, fn)
if err != nil {
log.Error(err, "Failed to update function serving status")
// in canary progress need to update canary serving
if inCanaryProgress(fn) {
if err := r.updateFuncWithServingStatus(fn, fn.Status.Serving); err != nil {
return err

}
return nil
}
if err := r.updateFuncWithServingStatus(fn, fn.Status.RolloutStatus.Canary.Serving); err != nil {
// sync to canary serving
fn.Status.RolloutStatus.Canary.Serving = fn.Status.Serving.DeepCopy()
if fn.Status.Revision != nil {
fn.Status.RolloutStatus.Canary.Revision = fn.Status.Revision.DeepCopy()
}
if err := r.Status().Update(r.ctx, fn); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -563,9 +577,10 @@ func (r *FunctionReconciler) createServing(fn *openfunction.Function) error {
return nil
}
servingSpec := r.createServingSpec(fn)
newServingHash := util.Hash(servingSpec)
// rollback
canary := fn.Status.RolloutStatus.Canary
if canary.Serving != nil && canary.Serving.ResourceHash == util.Hash(servingSpec) {
if inCanaryProgress(fn) && canary.Serving.ResourceHash == newServingHash {
fn.Status.Serving = canary.Serving
canary.CanaryStepStatus.Phase = openfunction.CanaryPhaseHealthy
canary.CanaryStepStatus.CurrentStepState = openfunction.CanaryStepStatePaused
Expand All @@ -576,12 +591,9 @@ func (r *FunctionReconciler) createServing(fn *openfunction.Function) error {
log.Error(err, "Failed to update function canary status")
return err
}
if err := r.cleanServing(fn); err != nil {
log.Error(err, "Failed to clean Serving")
return err
}
return nil
}

serving := &openfunction.Serving{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "serving-",
Expand Down Expand Up @@ -609,33 +621,31 @@ func (r *FunctionReconciler) createServing(fn *openfunction.Function) error {
return err
}
}
// initialize or reinitialize canary step
if inCanaryProgress(fn) || (hasCanaryReleasePlan(fn) && canary.Serving != nil) {
canary.CanaryStepStatus = &openfunction.CanaryStepStatus{
CurrentStepIndex: 0,
CurrentStepState: openfunction.CanaryStepStatePaused,
Message: "Canary release in progress",
LastUpdateTime: &metav1.Time{Time: time.Now()},
Phase: openfunction.CanaryPhaseProgressing,
}
}

servingStatus := &openfunction.Condition{
State: openfunction.Created,
ResourceRef: serving.Name,
ResourceHash: util.Hash(serving.Spec),
ResourceHash: newServingHash,
LastSuccessfulResourceRef: fn.Status.Serving.LastSuccessfulResourceRef,
}

fn.Status.Serving = servingStatus
canary = fn.Status.RolloutStatus.Canary

// when serving created,canary release should reinitialize
if hasCanaryReleasePlan(fn) && canary.Serving != nil {
canary.CanaryStepStatus = &openfunction.CanaryStepStatus{
CurrentStepIndex: 0,
CurrentStepState: openfunction.CanaryStepStatePaused,
Message: "Canary release in progress",
LastUpdateTime: &metav1.Time{Time: time.Now()},
Phase: openfunction.CanaryPhaseProgressing,
if !inCanaryProgress(fn) {
canary.Serving = fn.Status.Serving.DeepCopy()
if fn.Status.Revision != nil {
canary.Revision = fn.Status.Revision.DeepCopy()
}
}

// first time create serving or no need canary release
if canary.Serving == nil || !hasCanaryReleasePlan(fn) {
// create stable serving status
canary.Serving = servingStatus.DeepCopy()
canary.Revision = fn.Status.Revision.DeepCopy()
}
fn.Status.RolloutStatus.Canary = canary
if err := r.Status().Update(r.ctx, fn); err != nil {
Expand Down

0 comments on commit 2eaa968

Please sign in to comment.