diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 33a939538b9c3..41d20ef3ff9d7 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2715,6 +2715,11 @@ spec: description: EnvoyLog is not implemented and may be removed in the future. Setting this has no effect. type: string + etcdManaged: + description: 'EtcdManagd installs an additional etcd cluster + that is used for Cilium state change. The cluster is operated + by cilium-etcd-operator. Default: false' + type: boolean ipam: description: Ipam specifies the IP address allocation mode to use. Possible values are "crd" and "eni". "eni" will use AWS diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 68a7992878048..9299d4afb4e6c 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -402,6 +402,10 @@ type CiliumNetworkingSpec struct { // Requires spec.kubeProxy.enabled be set to false. // Default: false EnableNodePort bool `json:"enableNodePort"` + // EtcdManagd installs an additional etcd cluster that is used for Cilium state change. + // The cluster is operated by cilium-etcd-operator. + // Default: false + EtcdManaged bool `json:"etcdManaged,omitempty"` // RemoveCbrBridge is not implemented and may be removed in the future. // Setting this has no effect. diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go index 95e38a8cbe306..828f1df9c232b 100644 --- a/pkg/apis/kops/v1alpha1/networking.go +++ b/pkg/apis/kops/v1alpha1/networking.go @@ -400,6 +400,10 @@ type CiliumNetworkingSpec struct { // Requires spec.kubeProxy.enabled be set to false. // Default: false EnableNodePort bool `json:"enableNodePort"` + // EtcdManagd installs an additional etcd cluster that is used for Cilium state change. + // The cluster is operated by cilium-etcd-operator. + // Default: false + EtcdManaged bool `json:"etcdManaged,omitempty"` // RemoveCbrBridge is not implemented and may be removed in the future. // Setting this has no effect. diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index dbdbdfe0db127..12a0a505e1b78 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1334,6 +1334,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.EtcdManaged = in.EtcdManaged out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet @@ -1413,6 +1414,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.EtcdManaged = in.EtcdManaged out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index dea54ea1c066c..552eebfb9f38d 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -400,6 +400,10 @@ type CiliumNetworkingSpec struct { // Requires spec.kubeProxy.enabled be set to false. // Default: false EnableNodePort bool `json:"enableNodePort"` + // EtcdManagd installs an additional etcd cluster that is used for Cilium state change. + // The cluster is operated by cilium-etcd-operator. + // Default: false + EtcdManaged bool `json:"etcdManaged,omitempty"` // RemoveCbrBridge is not implemented and may be removed in the future. // Setting this has no effect. diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 876720e3939fb..000a6d2254c57 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1376,6 +1376,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.EtcdManaged = in.EtcdManaged out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet @@ -1455,6 +1456,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.IPTablesRulesNoinstall = in.IPTablesRulesNoinstall out.AutoDirectNodeRoutes = in.AutoDirectNodeRoutes out.EnableNodePort = in.EnableNodePort + out.EtcdManaged = in.EtcdManaged out.RemoveCbrBridge = in.RemoveCbrBridge out.RestartPods = in.RestartPods out.ReconfigureKubelet = in.ReconfigureKubelet diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template index 268f5342d9dd7..f904d2c817f9e 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template @@ -7,6 +7,21 @@ metadata: role.kubernetes.io/networking: "1" data: {{ with .Networking.Cilium }} + +{{- if .EtcdManaged }} + kvstore: etcd + kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config", "etcd.operator": "true"}' + + etcd-config: |- + --- + endpoints: + - https://cilium-etcd-client.kube-system.svc:2379 + + trusted-ca-file: '/var/lib/etcd-secrets/etcd-client-ca.crt' + key-file: '/var/lib/etcd-secrets/etcd-client.key' + cert-file: '/var/lib/etcd-secrets/etcd-client.crt' +{{ end }} + # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). @@ -460,6 +475,14 @@ spec: name: cni-path - mountPath: /host/etc/cni/net.d name: etc-cni-netd +{{ if .EtcdManaged }} + - mountPath: /var/lib/etcd-config + name: etcd-config-path + readOnly: true + - mountPath: /var/lib/etcd-secrets + name: etcd-secrets + readOnly: true +{{ end }} - mountPath: /var/lib/cilium/clustermesh name: clustermesh-secrets readOnly: true @@ -548,6 +571,22 @@ spec: type: FileOrCreate name: xtables-lock # To read the clustermesh configuration +{{- if .Networking.Cilium.EtcdManaged }} + # To read the etcd config stored in config maps + - configMap: + defaultMode: 420 + items: + - key: etcd-config + path: etcd.config + name: cilium-config + name: etcd-config-path + # To read the Cilium etcd secrets in case the user might want to use TLS + - name: etcd-secrets + secret: + defaultMode: 420 + optional: true + secretName: cilium-etcd-secrets +{{- end }} - name: clustermesh-secrets secret: defaultMode: 420 @@ -699,10 +738,37 @@ spec: initialDelaySeconds: 60 periodSeconds: 10 timeoutSeconds: 3 +{{- if .EtcdManaged }} + volumeMounts: + - mountPath: /var/lib/etcd-config + name: etcd-config-path + readOnly: true + - mountPath: /var/lib/etcd-secrets + name: etcd-secrets + readOnly: true +{{- end }} hostNetwork: true restartPolicy: Always serviceAccount: cilium-operator serviceAccountName: cilium-operator +{{- if .EtcdManaged }} + volumes: + # To read the etcd config stored in config maps + - configMap: + defaultMode: 420 + items: + - key: etcd-config + path: etcd.config + name: cilium-config + name: etcd-config-path + # To read the k8s etcd secrets in case the user might want to use TLS + - name: etcd-secrets + secret: + defaultMode: 420 + optional: true + secretName: cilium-etcd-secrets +{{- end }} + {{ if eq .Ipam "eni" }} nodeSelector: node-role.kubernetes.io/master: "" @@ -719,3 +785,241 @@ spec: tolerationSeconds: 300 {{ end }} {{ end }} + +{{ if .Networking.Cilium.EtcdManaged }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + role.kubernetes.io/networking: "1" + name: cilium-etcd-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-etcd-operator +subjects: +- kind: ServiceAccount + name: cilium-etcd-operator + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-etcd-operator +rules: +- apiGroups: + - etcd.database.coreos.com + resources: + - etcdclusters + verbs: + - get + - delete + - create + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - delete + - get + - create +- apiGroups: + - "" + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - get + - delete +- apiGroups: + - apps + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - componentstatuses + verbs: + - get +- apiGroups: + - extensions + resources: + - deployments + verbs: + - delete + - create + - get + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - delete +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + name: cilium-etcd-operator + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + io.cilium/app: etcd-operator + name: cilium-etcd-operator + spec: + containers: + - command: + - /usr/bin/cilium-etcd-operator + env: + - name: CILIUM_ETCD_OPERATOR_CLUSTER_DOMAIN + value: "{{ $.ClusterDNSDomain }}" + - name: CILIUM_ETCD_OPERATOR_ETCD_CLUSTER_SIZE + value: "3" + - name: CILIUM_ETCD_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_ETCD_OPERATOR_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: CILIUM_ETCD_OPERATOR_POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid + - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_MODE + value: "revision" + - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_RETENTION + value: "25000" + image: "cilium/cilium-etcd-operator:v2.0.7" + name: cilium-etcd-operator + dnsPolicy: ClusterFirst + hostNetwork: true + restartPolicy: Always + serviceAccount: cilium-etcd-operator + serviceAccountName: cilium-etcd-operator + tolerations: + - operator: Exists +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + role.kubernetes.io/networking: "1" + name: cilium-etcd-operator + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + role.kubernetes.io/networking: "1" + name: etcd-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: etcd-operator +subjects: +- kind: ServiceAccount + name: cilium-etcd-sa + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + role.kubernetes.io/networking: "1" + name: etcd-operator +rules: +- apiGroups: + - etcd.database.coreos.com + resources: + - etcdclusters + - etcdbackups + - etcdrestores + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - deployments + verbs: + - '*' +- apiGroups: + - apps + resources: + - deployments + verbs: + - '*' +- apiGroups: + - extensions + resources: + - deployments + verbs: + - create + - get + - list + - patch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + role.kubernetes.io/networking: "1" + name: cilium-etcd-sa + namespace: kube-system +{{ end }} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index c5e0c8106147c..fc43e42c6d094 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -931,7 +931,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { if b.cluster.Spec.Networking.Cilium != nil { key := "networking.cilium.io" - version := "1.7.0-kops.1" + version := "1.7.0-kops.2" { id := "k8s-1.7" diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index e4a4a96083052..e2ebaa6976197 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -93,7 +93,7 @@ spec: name: networking.cilium.io selector: role.kubernetes.io/networking: "1" - version: 1.7.0-kops.1 + version: 1.7.0-kops.2 - id: k8s-1.12 kubernetesVersion: '>=1.12.0' manifest: networking.cilium.io/k8s-1.12.yaml @@ -101,4 +101,4 @@ spec: name: networking.cilium.io selector: role.kubernetes.io/networking: "1" - version: 1.7.0-kops.1 + version: 1.7.0-kops.2