Merge pull request #71 from alibaba/tengine-ingress-3.1.0

HTTP Route: supports routing based on multiple values, nginx var, or mod of header, cookie, query parameter while also being able to add HTTP headers and query parameter

internal/ingress/annotations/canary/main.go

@@ -19,24 +19,102 @@ package canary
 
 import (
 	networking "k8s.io/api/networking/v1"
+	"k8s.io/klog"
 
 	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
 	"k8s.io/ingress-nginx/internal/ingress/errors"
 	"k8s.io/ingress-nginx/internal/ingress/resolver"
 )
 
+const (
+	// A list of routing priority for canary ingresses
+	CanaryPriorityList = "canary-priority-list"
+	// Enable or disable canary ingress
+	CanaryFlag = "canary"
+	// Canary routing based on header with value 'always'
+	CanaryByHeader = "canary-by-header"
+	// Canary routing based on header with specific values
+	// Format: <header value>[||<header value>]*
+	// Default max number of header value is 20
+	CanaryByHeaderVal = "canary-by-header-value"
+	// Canary routing based on cookie with value 'always'
+	CanaryByCookie = "canary-by-cookie"
+	// Canary routing based on cookie with specific values
+	// Format: <cookie value>[||<cookie value>]*
+	// Default max number cookie value is 20
+	CanaryByCookieVal = "canary-by-cookie-value"
+	// Canary routing based on query with value 'always'
+	CanaryByQuery = "canary-by-query"
+	// Canary routing based on query with specific values
+	// Format: <query value>[||<query value>]*
+	// Default max number query value is 20
+	CanaryByQueryVal = "canary-by-query-value"
+	// Mod divisor
+	CanaryModDivisor = "canary-mod-divisor"
+	// Mod relational operator
+	CanaryModRelationalOpr = "canary-mod-relational-operator"
+	// Mod remainder
+	CanaryModRemainder = "canary-mod-remainder"
+	// Canary weight
+	// range: [0, CanaryWeightTotal]
+	CanaryWeight = "canary-weight"
+	// canary weight total
+	// Default range: [100, 10000]
+	CanaryWeightTotal = "canary-weight-total"
+	// Add header to request based on canary ingress
+	// Format: <header name>:<header value>[||<header name>:<header value>]*
+	// Default max number header is 2
+	// If the header is present on the request, the header and value will be added to the request again.
+	CanaryReqAddHeader = "canary-request-add-header"
+	// Append header value to request header based on canary ingress
+	// Format: <header name>:<header value>[||<header name>:<header value>]*
+	// Default max number header is 2
+	// If the header is not present on the request, the header will be added to the request.
+	CanaryReqAppendHeader = "canary-request-append-header"
+	// Add query to request based on canary ingress
+	// Format: <query name>=<query value>[&<query name>=<query value>]*
+	// Default max number query is 2
+	// If the query is present on the request, the query and value will be added to the request again.
+	CanaryReqAddQuery = "canary-request-add-query"
+	// Add header to response based on canary ingress
+	// Format: <header name>:<header value>[||<header name>:<header value>]*
+	// Default max number header is 2
+	// If the header is present on the request, the header and value will be added to the response again.
+	CanaryRespAddHeader = "canary-response-add-header"
+	// Append header value to response header based on canary ingress
+	// Format: <header name>:<header value>[||<header name>:<header value>]*
+	// Default max number header is 2
+	// If the header is not present on the request, the header will be added to the response.
+	CanaryRespAppendHeader = "canary-response-append-header"
+	// Referrer of canary ingress
+	CanaryReferrer = "canary-referrer"
+)
+
 type canary struct {
 	r resolver.Resolver
 }
 
 // Config returns the configuration rules for setting up the Canary
 type Config struct {
-	Enabled     bool
-	Weight      int
-	Header      string
-	HeaderValue string
-	Cookie      string
-	Referrer    string
+	Enabled          bool
+	Weight           int
+	WeightTotal      int
+	Header           string
+	HeaderValue      string
+	Cookie           string
+	CookieValue      string
+	Query            string
+	QueryValue       string
+	ModDivisor       int
+	ModRelationalOpr string
+	ModRemainder     int
+	ReqAddHeader     string
+	ReqAppendHeader  string
+	ReqAddQuery      string
+	RespAddHeader    string
+	RespAppendHeader string
+	Priority         string
+	Referrer         string
 }
 
 // NewParser parses the ingress for canary related annotations
@@ -50,39 +128,117 @@ func (c canary) Parse(ing *networking.Ingress) (interface{}, error) {
 	config := &Config{}
 	var err error
 
-	config.Enabled, err = parser.GetBoolAnnotation("canary", ing)
+	config.Enabled, err = parser.GetBoolAnnotation(CanaryFlag, ing)
 	if err != nil {
 		config.Enabled = false
 	}
 
-	config.Weight, err = parser.GetIntAnnotation("canary-weight", ing)
+	config.Weight, err = parser.GetIntAnnotation(CanaryWeight, ing)
 	if err != nil {
 		config.Weight = 0
 	}
 
-	config.Header, err = parser.GetStringAnnotation("canary-by-header", ing)
+	config.WeightTotal, err = parser.GetIntAnnotation(CanaryWeightTotal, ing)
+	if err != nil {
+		config.WeightTotal = 100
+	}
+
+	config.Header, err = parser.GetStringAnnotation(CanaryByHeader, ing)
 	if err != nil {
 		config.Header = ""
 	}
 
-	config.HeaderValue, err = parser.GetStringAnnotation("canary-by-header-value", ing)
+	config.HeaderValue, err = parser.GetStringAnnotation(CanaryByHeaderVal, ing)
 	if err != nil {
 		config.HeaderValue = ""
 	}
 
-	config.Cookie, err = parser.GetStringAnnotation("canary-by-cookie", ing)
+	config.Cookie, err = parser.GetStringAnnotation(CanaryByCookie, ing)
 	if err != nil {
 		config.Cookie = ""
 	}
 
-	config.Referrer, err = parser.GetStringAnnotation("canary-referrer", ing)
+	config.CookieValue, err = parser.GetStringAnnotation(CanaryByCookieVal, ing)
+	if err != nil {
+		config.CookieValue = ""
+	}
+
+	config.Query, err = parser.GetStringAnnotation(CanaryByQuery, ing)
+	if err != nil {
+		config.Query = ""
+	}
+
+	config.QueryValue, err = parser.GetStringAnnotation(CanaryByQueryVal, ing)
+	if err != nil {
+		config.QueryValue = ""
+	}
+
+	config.ModDivisor, err = parser.GetIntAnnotation(CanaryModDivisor, ing)
+	if err != nil {
+		config.ModDivisor = 0
+	}
+
+	config.ModRelationalOpr, err = parser.GetStringAnnotation(CanaryModRelationalOpr, ing)
+	if err != nil {
+		config.ModRelationalOpr = ""
+	}
+
+	config.ModRemainder, err = parser.GetIntAnnotation(CanaryModRemainder, ing)
+	if err != nil {
+		config.ModRemainder = 0
+	}
+
+	config.ReqAddHeader, err = parser.GetStringAnnotation(CanaryReqAddHeader, ing)
+	if err != nil {
+		config.ReqAddHeader = ""
+	}
+
+	config.ReqAppendHeader, err = parser.GetStringAnnotation(CanaryReqAppendHeader, ing)
+	if err != nil {
+		config.ReqAppendHeader = ""
+	}
+
+	config.ReqAddQuery, err = parser.GetStringAnnotation(CanaryReqAddQuery, ing)
+	if err != nil {
+		config.ReqAddQuery = ""
+	}
+
+	config.RespAddHeader, err = parser.GetStringAnnotation(CanaryRespAddHeader, ing)
+	if err != nil {
+		config.RespAddHeader = ""
+	}
+
+	config.RespAppendHeader, err = parser.GetStringAnnotation(CanaryRespAppendHeader, ing)
+	if err != nil {
+		config.RespAppendHeader = ""
+	}
+
+	config.Referrer, err = parser.GetStringAnnotation(CanaryReferrer, ing)
 	if err != nil {
 		config.Referrer = ""
 	}
 
-	if !config.Enabled && (config.Weight > 0 || len(config.Header) > 0 || len(config.HeaderValue) > 0 || len(config.Cookie) > 0) {
+	config.Priority, err = parser.GetStringAnnotation(CanaryPriorityList, ing)
+	if err != nil {
+		config.Priority = ""
+	}
+
+	if !config.Enabled &&
+		(config.Weight > 0 ||
+			len(config.Header) > 0 ||
+			len(config.HeaderValue) > 0 ||
+			len(config.Cookie) > 0 ||
+			len(config.CookieValue) > 0 ||
+			len(config.Query) > 0 ||
+			len(config.QueryValue) > 0) {
+		klog.Warningf("Canary ingress[%v/%v] configured but not enabled, ignored", ing.Namespace, ing.Name)
 		return nil, errors.NewInvalidAnnotationConfiguration("canary", "configured but not enabled")
 	}
 
+	if config.Enabled && len(config.Referrer) == 0 {
+		klog.Warningf("Canary ingress[%v/%v] with empty referrer, ignored", ing.Namespace, ing.Name)
+		return nil, errors.NewInvalidAnnotationConfiguration("canary", "referrer is empty")
+	}
+
 	return config, nil
 }

internal/ingress/controller/config/config.go

@@ -674,9 +674,6 @@ type Configuration struct {
 	// File path of status tengine
 	StatusTengineFilePath string `json:"filepath-status-tengine"`
 
-	// Max canary ingress number
-	MaxCanaryIngNum int `json:"max-canary-ing-num"`
-
 	// Canary referrer: this is a multi-valued field, separated by ','
 	CanaryReferrer string `json:"canary-referrer"`
 
@@ -718,6 +715,45 @@ type Configuration struct {
 	// Default HTTP3/XQUIC port for clients
 	HTTP3xQUICDefaultPort int `json:"http3-xquic-default-port"`
 
+	// Max number of path for the same host
+	MaxHostPathNum int `json:"max-host-path-num"`
+
+	// Max canary ingress number
+	MaxCanaryIngNum int `json:"max-canary-ing-num"`
+
+	// Max number of action for canary ingress with header and query
+	MaxCanaryActionNum int `json:"max-canary-action-num"`
+
+	// The default total weight of traffic
+	DefaultCanaryWeightTotal uint32 `json:"default-canary-weight-total"`
+
+	// The max total weight of traffic
+	MaxCanaryWeightTotal uint32 `json:"max-canary-weight-total"`
+
+	// Max number of header value for canary annotation 'canary-by-header-value'
+	MaxCanaryHeaderValNum int `json:"max-canary-header-val-num"`
+
+	// Max number of cookie value for canary annotation 'canary-by-cookie-value'
+	MaxCanaryCookieValNum int `json:"max-canary-cookie-val-num"`
+
+	// Max number of query value for canary annotation 'canary-by-query-value'
+	MaxCanaryQueryValNum int `json:"max-canary-query-val-num"`
+
+	// Max number of adding fields to the request header for canary annotation 'canary-request-add-header'
+	MaxReqAddHeaderNum int `json:"max-canary-req-add-header-num"`
+
+	// Max number of appending fields to the request header for canary annotation 'canary-request-append-header'
+	MaxReqAppendHeaderNum int `json:"max-canary-req-append-header-num"`
+
+	// Max number of adding fields to the request query for canary annotation 'canary-request-add-query'
+	MaxReqAddQueryNum int `json:"max-canary-req-add-query-num"`
+
+	// Max number of adding fields to the response header for canary annotation 'canary-response-add-header'
+	MaxRespAddHeaderNum int `json:"max-canary-resp-add-header-num"`
+
+	// Max number of appending fields to the response header for canary annotation 'canary-response-append-header'
+	MaxRespAppendHeaderNum int `json:"max-canary-resp-append-header-num"`
+
 	// Set user of Tengine worker processes
 	User string `json:"user"`
 }
@@ -870,7 +906,6 @@ func NewDefault() Configuration {
 		TengineStaticServiceCfg:      false,
 		ShmServiceCfgFileLock:        "/etc/nginx/shm_service_cfg.lock",
 		StatusTengineFilePath:        "/etc/nginx/htdocs/status.tengine",
-		MaxCanaryIngNum:              200,
 		CanaryReferrer:               "",
 		IngressReferrer:              "",
 		UseCustomDefBackend:          true,
@@ -884,6 +919,19 @@ func NewDefault() Configuration {
 		HTTP3xQUICDefaultCert:        "",
 		HTTP3xQUICDefaultKey:         "",
 		HTTP3xQUICDefaultPort:        443,
+		MaxHostPathNum:               20,
+		MaxCanaryIngNum:              20,
+		MaxCanaryActionNum:           10,
+		DefaultCanaryWeightTotal:     100,
+		MaxCanaryWeightTotal:         10000,
+		MaxCanaryHeaderValNum:        20,
+		MaxCanaryCookieValNum:        20,
+		MaxCanaryQueryValNum:         20,
+		MaxReqAddHeaderNum:           2,
+		MaxReqAppendHeaderNum:        2,
+		MaxReqAddQueryNum:            2,
+		MaxRespAddHeaderNum:          2,
+		MaxRespAppendHeaderNum:       2,
 		User:                         "root",
 	}
 

internal/ingress/controller/controller.go

@@ -836,12 +836,7 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
 			// configure traffic shaping for canary
 			if anns.Canary.Enabled && n.verifyCanaryReferrer(ingKey, anns) {
 				upstreams[defBackend].NoServer = true
-				upstreams[defBackend].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
-					Weight:      anns.Canary.Weight,
-					Header:      anns.Canary.Header,
-					HeaderValue: anns.Canary.HeaderValue,
-					Cookie:      anns.Canary.Cookie,
-				}
+				setTrafficShapingPolicy(anns, &upstreams[defBackend].TrafficShapingPolicy)
 			}
 
 			if len(upstreams[defBackend].Endpoints) == 0 {
@@ -904,13 +899,8 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
 
 				// configure traffic shaping for canary
 				if anns.Canary.Enabled && n.verifyCanaryReferrer(ingKey, anns) {
-					upstreams[name].NoServer = true
-					upstreams[name].TrafficShapingPolicy = ingress.TrafficShapingPolicy{
-						Weight:      anns.Canary.Weight,
-						Header:      anns.Canary.Header,
-						HeaderValue: anns.Canary.HeaderValue,
-						Cookie:      anns.Canary.Cookie,
-					}
+					upstreams[svcName].NoServer = true
+					setTrafficShapingPolicy(anns, &upstreams[svcName].TrafficShapingPolicy)
 				}
 
 				if len(upstreams[name].Endpoints) == 0 {
@@ -1646,3 +1636,23 @@ func (n *NGINXController) verifyCanaryReferrer(ingKey string, anns *annotations.
 	klog.Warningf("Canary ingress[%v] with referrer [%v] is illegal, ignored", ingKey, anns.Canary.Referrer)
 	return false
 }
+
+func setTrafficShapingPolicy(anns *annotations.Ingress, policy *ingress.TrafficShapingPolicy) {
+	*policy = ingress.TrafficShapingPolicy{
+		Weight:           anns.Canary.Weight,
+		Header:           anns.Canary.Header,
+		HeaderValue:      anns.Canary.HeaderValue,
+		Cookie:           anns.Canary.Cookie,
+		CookieValue:      anns.Canary.CookieValue,
+		Query:            anns.Canary.Query,
+		QueryValue:       anns.Canary.QueryValue,
+		ModDivisor:       uint64(anns.Canary.ModDivisor),
+		ModRelationalOpr: anns.Canary.ModRelationalOpr,
+		ModRemainder:     uint64(anns.Canary.ModRemainder),
+		ReqAddHeader:     anns.Canary.ReqAddHeader,
+		ReqAppendHeader:  anns.Canary.ReqAppendHeader,
+		ReqAddQuery:      anns.Canary.ReqAddQuery,
+		RespAddHeader:    anns.Canary.RespAddHeader,
+		RespAppendHeader: anns.Canary.RespAppendHeader,
+	}
+}