Merge remote-tracking branch 'origin/staging' into handle-maintenance…

…-mode

.github/rvu/labels.yaml

@@ -0,0 +1 @@
+service.rvu.co.uk/brand: airship

.github/workflows/push.yaml

@@ -0,0 +1,57 @@
+name: push
+on: push
+permissions:
+  contents: read
+  id-token: write
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: "1.17"
+    - run: make test
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: "1.17"
+    - run: make build-linux
+    - uses: actions/upload-artifact@v3
+      with:
+        name: bin
+        path: bin/
+  docker-build-push:
+    if: github.ref_name == 'master' || startsWith(github.ref, 'refs/tags/v')
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/download-artifact@v3
+      with:
+        name: bin
+        path: bin/
+    - name: Login to Quay.io
+      uses: docker/login-action@v3
+      with:
+        registry: quay.io
+        username: ${{ secrets.QUAY_USERNAME }}
+        password: ${{ secrets.QUAY_PASSWORD }}
+    - id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: quay.io/uswitch/yggdrasil
+        tags: |
+          type=semver,pattern={{raw}}
+          type=sha,prefix=,format=long,
+    - uses: docker/build-push-action@v5
+      with:
+        context: .
+        labels: ${{ steps.meta.outputs.labels }}
+        push: true
+        tags: ${{ steps.meta.outputs.tags }}
+

.tool-versions

@@ -0,0 +1 @@
+golang 1.17.13

Dockerfile

@@ -1,5 +1,5 @@
 FROM scratch
 
-ADD bin/yggdrasil-linux-amd64 yggdrasil
+COPY --chmod=755 bin/yggdrasil-linux-amd64 yggdrasil
 
 ENTRYPOINT ["/yggdrasil"]

Makefile

@@ -5,14 +5,16 @@ BIN_DARWIN = $(BIN)-darwin-$(ARCH)
 
 SOURCES := $(shell find . -iname '*.go')
 
-.PHONY: test clean all
+.PHONY: test clean all build-linux
 
-all: build-darwin build-linux
+all: build-darwin $(BIN_LINUX)
 
 build-darwin: $(SOURCES)
 	GOARCH=$(ARCH) GOOS=darwin go build -o $(BIN_DARWIN)
 
-build-linux: $(SOURCES)
+build-linux: $(BIN_LINUX)
+
+$(BIN_LINUX): $(SOURCES)
 	GOARCH=$(ARCH) GOOS=linux CGO_ENABLED=0 go build -o $(BIN_LINUX)
 
 test: $(SOURCES)
@@ -22,7 +24,7 @@ bench: $(SOURCES)
 	go test -run=XX -bench=. $(shell go list ./... | grep -v /vendor)
 
 docker: Dockerfile $(BIN_LINUX)
-	docker image build -t quay.io/uswitch/yggdrasil:devel .
+	docker image build -t registry.airship.rvu.cloud/cloud/yggdrasil:devel .
 
 clean:
 	rm -rf bin/

README.md

@@ -1,7 +1,7 @@
 # Yggdrasil
 Yggdrasil is an Envoy control plane that configures listeners and clusters based off Kubernetes ingresses from multiple Kube Clusters. This allows you to have an envoy cluster acting as a mutli-cluster loadbalancer for Kubernetes. This was something we needed as we wanted our apps to be highly available in the event of a cluster outage but did not want the solution to live inside of Kubernetes itself.
 
-`Note:` Currently we support version 1.19.x of Envoy.</br>
+`Note:` Currently we support versions 1.20.x to 1.26.x of Envoy.</br>
 `Note:` Yggdrasil now uses [Go modules](https://github.com/golang/go/wiki/Modules) to handle dependencies.
 
 ## Usage
@@ -133,12 +133,22 @@ spec:
           servicePort: 80
 ```
 
+## Dynamic TLS certificates synchronization from Kubernetes secrets
+
+Downstream TLS certificates can be dynamically fetched and updated from Kubernetes secrets configured under ingresses' `spec.tls` by setting `syncSecrets` true in Yggdrasil configuration (false by default).
+
+In this mode, only a single `certificate` may be specified in Yggdrasil configuration. It will be used for hosts with misconfigured or invalid secret.
+
+**Note**: ECDSA >256 keys are not supported by envoy and will be discarded. See https://github.com/envoyproxy/envoy/issues/10855
+
 ## Configuration
 Yggdrasil can be configured using a config file e.g:
 ```json
 {
   "nodeName": "foo",
   "ingressClasses": ["multi-cluster", "multi-cluster-staging"],
+  "accessLog": "/var/log/envoy/",
+  "syncSecrets": false,
   "certificates": [
     {
       "hosts": ["*.api.com"],
@@ -199,17 +209,19 @@ The Yggdrasil-specific metrics which are available from the API are:
 --ca string                                   trustedCA
 --cert string                                 certfile
 --config string                               config file
+--config-dump                                 Enable config dump endpoint at /configdump on the health-address HTTP server
 --debug                                       Log at debug level
+--access-log                                  path for the file logs
 --envoy-listener-ipv4-address strings         IPv4 addresses by the envoy proxy to accept incoming connections (default "0.0.0.0")
 --envoy-port uint32                           port by the envoy proxy to accept incoming connections (default 10000)
 --health-address string                       yggdrasil health API listen address (default "0.0.0.0:8081")
---help                                        help for yggdrasil
+-h, --help                                        help for yggdrasil
 --host-selection-retry-attempts int           Number of host selection retry attempts. Set to value >=0 to enable (default -1)
---retry-on                                    Default comma-separated list of retry policies (default 5xx)
 --http-ext-authz-allow-partial-message        When this field is true, Envoy will buffer the message until max_request_bytes is reached (default true)
 --http-ext-authz-cluster string               The name of the upstream gRPC cluster
 --http-ext-authz-failure-mode-allow           Changes filters behaviour on errors (default true)
 --http-ext-authz-max-request-bytes uint32     Sets the maximum size of a message body that the filter will hold in memory (default 8192)
+--http-ext-authz-pack-as-bytes                When this field is true, Envoy will send the body as raw bytes.
 --http-ext-authz-timeout duration             The timeout for the gRPC request. This is the timeout for a specific request. (default 200ms)
 --http-grpc-logger-cluster string             The name of the upstream gRPC cluster
 --http-grpc-logger-name string                Name of the access log
@@ -221,6 +233,8 @@ The Yggdrasil-specific metrics which are available from the API are:
 --kube-config stringArray                     Path to kube config
 --max-ejection-percentage int32               maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy. (default -1)
 --node-name string                            envoy node name
+--retry-on string                             default comma-separated list of retry policies (default "5xx")
+--tracing-provider                            name of HTTP Connection Manager tracing provider to include - currently only zipkin config is supported
 --upstream-healthcheck-healthy uint32         number of successful healthchecks before the backend is considered healthy (default 3)
 --upstream-healthcheck-interval duration      duration of the upstream health check interval (default 10s)
 --upstream-healthcheck-timeout duration       timeout of the upstream healthchecks (default 5s)

cmd/root.go

@@ -36,6 +36,7 @@ type config struct {
 	NodeName                   string                    `json:"nodeName"`
 	Clusters                   []clusterConfig           `json:"clusters"`
 	SyncSecrets                bool                      `json:"syncSecrets"`
+	AccessLog                  string                    `json:"accessLog"`
 	Certificates               []envoy.Certificate       `json:"certificates"`
 	TrustCA                    string                    `json:"trustCA"`
 	UpstreamPort               uint32                    `json:"upstreamPort"`
@@ -49,6 +50,7 @@ type config struct {
 	HttpGrpcLogger             envoy.HttpGrpcLogger      `json:"httpGrpcLogger"`
 	DefaultTimeouts            envoy.DefaultTimeouts     `json:"defaultTimeouts"`
 	AlpnProtocols              []string                  `json:"alpnProtocols"`
+	AccessLogger               envoy.AccessLogger        `json:"accessLogger"`
 }
 
 // Hasher returns node ID as an ID
@@ -80,18 +82,21 @@ func init() {
 	rootCmd.PersistentFlags().String("address", "0.0.0.0:8080", "yggdrasil envoy control plane listen address")
 	rootCmd.PersistentFlags().String("health-address", "0.0.0.0:8081", "yggdrasil health API listen address")
 	rootCmd.PersistentFlags().String("node-name", "", "envoy node name")
+	rootCmd.PersistentFlags().String("access-log", "/var/log/envoy/", "envoy default access log file")
 	rootCmd.PersistentFlags().String("cert", "", "certfile")
 	rootCmd.PersistentFlags().String("key", "", "keyfile")
 	rootCmd.PersistentFlags().String("ca", "", "trustedCA")
 	rootCmd.PersistentFlags().StringSlice("ingress-classes", nil, "Ingress classes to watch")
 	rootCmd.PersistentFlags().StringArrayVar(&kubeConfig, "kube-config", nil, "Path to kube config")
 	rootCmd.PersistentFlags().Bool("debug", false, "Log at debug level")
+	rootCmd.PersistentFlags().Bool("config-dump", false, "Enable config dump endpoint at /configdump on the health-address HTTP server")
 	rootCmd.PersistentFlags().Uint32("upstream-port", 443, "port used to connect to the upstream ingresses")
 	rootCmd.PersistentFlags().StringSlice("envoy-listener-ipv4-address", []string{"0.0.0.0"}, "IPv4 address by the envoy proxy to accept incoming connections")
 	rootCmd.PersistentFlags().Uint32("envoy-port", 10000, "port by the envoy proxy to accept incoming connections")
 	rootCmd.PersistentFlags().Int32("max-ejection-percentage", -1, "maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy.")
 	rootCmd.PersistentFlags().Int64("host-selection-retry-attempts", -1, "Number of host selection retry attempts. Set to value >=0 to enable")
 	rootCmd.PersistentFlags().String("retry-on", "5xx", "default comma-separated list of retry policies")
+	rootCmd.PersistentFlags().String("tracing-provider", "", "HTTP Connection Manager tracing provider block to include")
 	rootCmd.PersistentFlags().Duration("upstream-healthcheck-interval", 10*time.Second, "duration of the upstream health check interval")
 	rootCmd.PersistentFlags().Duration("upstream-healthcheck-timeout", 5*time.Second, "timeout of the upstream healthchecks")
 	rootCmd.PersistentFlags().Uint32("upstream-healthcheck-healthy", 3, "number of successful healthchecks before the backend is considered healthy")
@@ -106,15 +111,19 @@ func init() {
 	rootCmd.PersistentFlags().Duration("http-ext-authz-timeout", 200*time.Millisecond, "The timeout for the gRPC request. This is the timeout for a specific request.")
 	rootCmd.PersistentFlags().Uint32("http-ext-authz-max-request-bytes", 8192, "Sets the maximum size of a message body that the filter will hold in memory")
 	rootCmd.PersistentFlags().Bool("http-ext-authz-allow-partial-message", true, "When this field is true, Envoy will buffer the message until max_request_bytes is reached")
+	rootCmd.PersistentFlags().Bool("http-ext-authz-pack-as-bytes", false, "When this field is true, Envoy will send the body as raw bytes.")
 	rootCmd.PersistentFlags().Bool("http-ext-authz-failure-mode-allow", true, "Changes filters behaviour on errors")
+
 	rootCmd.PersistentFlags().Duration("default-route-timeout", 15*time.Second, "Default timeout of the routes")
 	rootCmd.PersistentFlags().Duration("default-cluster-timeout", 30*time.Second, "Default timeout of the cluster")
 	rootCmd.PersistentFlags().Duration("default-per-try-timeout", 5*time.Second, "Default timeout of PerTry")
 	rootCmd.PersistentFlags().StringSlice("alpn-protocols", []string{}, "exposed listener ALPN protocols")
 	viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))
+	viper.BindPFlag("configDump", rootCmd.PersistentFlags().Lookup("config-dump"))
 	viper.BindPFlag("address", rootCmd.PersistentFlags().Lookup("address"))
 	viper.BindPFlag("healthAddress", rootCmd.PersistentFlags().Lookup("health-address"))
 	viper.BindPFlag("nodeName", rootCmd.PersistentFlags().Lookup("node-name"))
+	viper.BindPFlag("accessLog", rootCmd.PersistentFlags().Lookup("access-log"))
 	viper.BindPFlag("ingressClasses", rootCmd.PersistentFlags().Lookup("ingress-classes"))
 	viper.BindPFlag("cert", rootCmd.PersistentFlags().Lookup("cert"))
 	viper.BindPFlag("key", rootCmd.PersistentFlags().Lookup("key"))
@@ -125,6 +134,7 @@ func init() {
 	viper.BindPFlag("maxEjectionPercentage", rootCmd.PersistentFlags().Lookup("max-ejection-percentage"))
 	viper.BindPFlag("hostSelectionRetryAttempts", rootCmd.PersistentFlags().Lookup("host-selection-retry-attempts"))
 	viper.BindPFlag("retryOn", rootCmd.PersistentFlags().Lookup("retry-on"))
+	viper.BindPFlag("tracingProvider", rootCmd.PersistentFlags().Lookup("tracing-provider"))
 	viper.BindPFlag("upstreamHealthCheck.interval", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-interval"))
 	viper.BindPFlag("upstreamHealthCheck.timeout", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-timeout"))
 	viper.BindPFlag("upstreamHealthCheck.healthyThreshold", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-healthy"))
@@ -139,6 +149,7 @@ func init() {
 	viper.BindPFlag("httpExtAuthz.timeout", rootCmd.PersistentFlags().Lookup("http-ext-authz-timeout"))
 	viper.BindPFlag("httpExtAuthz.maxRequestBytes", rootCmd.PersistentFlags().Lookup("http-ext-authz-max-request-bytes"))
 	viper.BindPFlag("httpExtAuthz.allowPartialMessage", rootCmd.PersistentFlags().Lookup("http-ext-authz-allow-partial-message"))
+	viper.BindPFlag("httpExtAuthz.packAsBytes", rootCmd.PersistentFlags().Lookup("http-ext-authz-pack-as-bytes"))
 	viper.BindPFlag("httpExtAuthz.FailureModeAllow", rootCmd.PersistentFlags().Lookup("http-ext-authz-failure-mode-allow"))
 	viper.BindPFlag("defaultTimeouts.Route", rootCmd.PersistentFlags().Lookup("default-route-timeout"))
 	viper.BindPFlag("defaultTimeouts.Cluster", rootCmd.PersistentFlags().Lookup("default-cluster-timeout"))
@@ -234,6 +245,7 @@ func main(*cobra.Command, []string) error {
 		c.Certificates,
 		viper.GetString("trustCA"),
 		viper.GetStringSlice("ingressClasses"),
+		viper.GetString("accessLog"),
 		envoy.WithUpstreamPort(uint32(viper.GetInt32("upstreamPort"))),
 		envoy.WithEnvoyListenerIpv4Address(viper.GetStringSlice("envoyListenerIpv4Address")),
 		envoy.WithEnvoyPort(uint32(viper.GetInt32("envoyPort"))),
@@ -246,15 +258,18 @@ func main(*cobra.Command, []string) error {
 		envoy.WithSyncSecrets(c.SyncSecrets),
 		envoy.WithDefaultTimeouts(c.DefaultTimeouts),
 		envoy.WithDefaultRetryOn(viper.GetString("retryOn")),
+		envoy.WithAccessLog(c.AccessLogger),
+		envoy.WithTracingProvider(viper.GetString("tracingProvider")),
 		envoy.WithAlpnProtocols(viper.GetStringSlice("alpnProtocols")),
 	)
+	configurator.ValidateAndFormatPath()
 	snapshotter := envoy.NewSnapshotter(envoyCache, configurator, aggregator)
 
 	go snapshotter.Run(aggregator)
 	go aggregator.Run()
 
 	envoyServer := server.NewServer(ctx, envoyCache, &callbacks{})
-	go runEnvoyServer(envoyServer, viper.GetString("address"), viper.GetString("healthAddress"), ctx.Done())
+	go runEnvoyServer(envoyServer, snapshotter, viper.GetBool("configDump"), viper.GetString("address"), viper.GetString("healthAddress"), ctx.Done())
 
 	<-stopCh
 	return nil

cmd/server.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -16,6 +17,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+
+	"github.com/uswitch/yggdrasil/pkg/envoy"
 )
 
 type callbacks struct {
@@ -49,7 +52,7 @@ func (c *callbacks) OnFetchResponse(*discovery.DiscoveryRequest, *discovery.Disc
 	c.fetchResp++
 }
 
-func runEnvoyServer(envoyServer server.Server, address string, healthAddress string, stopCh <-chan struct{}) {
+func runEnvoyServer(envoyServer server.Server, snapshotter *envoy.Snapshotter, enableConfigDump bool, address string, healthAddress string, stopCh <-chan struct{}) {
 
 	grpcServer := grpc.NewServer(
 		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
@@ -76,6 +79,9 @@ func runEnvoyServer(envoyServer server.Server, address string, healthAddress str
 
 	healthMux.Handle("/metrics", promhttp.Handler())
 	healthMux.HandleFunc("/healthz", health)
+	if enableConfigDump {
+		healthMux.HandleFunc("/configdump", handleConfigDump(snapshotter))
+	}
 
 	go func() {
 		if err = grpcServer.Serve(lis); err != nil {
@@ -97,3 +103,34 @@ func runEnvoyServer(envoyServer server.Server, address string, healthAddress str
 func health(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(200)
 }
+
+type ConfigDumpError struct {
+	Error   error
+	Message string
+}
+
+func handleConfigDump(snapshotter *envoy.Snapshotter) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+
+		snapshot, err := snapshotter.ConfigDump()
+		if err != nil {
+			respErr := ConfigDumpError{
+				Error:   err,
+				Message: "Unable to get current snapshot from snapshotter, see error for details.",
+			}
+
+			w.Header().Add("Content-Type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			json.NewEncoder(w).Encode(respErr)
+			return
+		}
+
+		w.Header().Add("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(snapshot)
+	}
+}