*: add Alibaba Cloud Object Storage Service (OSS) backend for etcd-ba…

…ckup/restore-operators (#2065)

Gopkg.toml

@@ -73,3 +73,7 @@ required = [
  [[prune.project]]
  name = "k8s.io/code-generator"
  non-go = false
+
+[[constraint]]
+ name = "github.com/aliyun/aliyun-oss-go-sdk"
+ version = "=1.9.4"

doc/design/oss_backup.md

@@ -0,0 +1,42 @@
+# Backups using Alibaba Cloud Object Storage Service (OSS)
+
+## Cluster configured with OSS backup
+
+To create a backup in OSS, set `backup.storageType` to `"OSS"`, supply the path (in the format `<oss-bucket-name>/<path-to-backup-object>`) in `oss.path` and provide the Kubernetes secret storing the Alibaba Cloud account credentials to `oss.ossSecret`. The secret must exist prior to backup creation. Etcd backup operator will create the bucket and object if not found. The field `oss.endpoint` is the target OSS service endpoint where the data is backed up. By default, `http://oss-cn-hangzhou.aliyuncs.com` will be used. If you want to back up the data to other regions, please specify another endpoint from [the list of region endpoints](https://www.alibabacloud.com/help/doc-detail/31837.htm).
+
+
+An example backup manifest would look like:
+
+```yaml
+apiVersion: etcd.database.coreos.com/v1beta2
+kind: EtcdBackup
+metadata:
+ name: etcd-cluster-with-oss-backup
+ namespace: my-namespace
+spec:
+ backupPolicy:
+ ...
+ etcdEndpoints:
+ - "http://example-etcd-cluster-client:2379"
+ storageType: OSS
+ oss:
+ endpoint: http://oss-cn-hangzhou.aliyuncs.com
+ ossSecret: my-oss-credentials
+ path: my-etcd-backups-bucket/etcd.backup
+```
+
+### In Detail:
+
+- `"ossSecret"` represents the name of the Kubernetes secret object that stores the credentials needed for Alibaba Cloud authorization, namely an authorization token.
+
+ The Kubernetes secret manifest must have the following format:
+ ```yaml
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ name: my-oss-credentials
+ type: Opaque
+ data:
+ accessKeyID: <my-access-key-id>
+ accessKeySecret: <my-access-key-secret>
+ ```

doc/user/oss_backup.md

@@ -0,0 +1,125 @@
+# Backups using Alibaba Cloud Object Storage Service (OSS)
+
+Etcd backup operator backs up the data of an etcd cluster running on Kubernetes to a remote storage such as Alibaba Cloud Object Storage Service (OSS). If it is not deployed yet, please follow the [instructions](walkthrough/backup-operator.md#deploy-etcd-backup-operator) to deploy it, e.g. by running
+
+```sh
+kubectl apply -f example/etcd-backup-operator/deployment.yaml
+```
+
+## Setup Alibaba Cloud backup account, OSS bucket, and Secret
+
+1. Login [Alibaba Cloud Console](https://www.alibabacloud.com) (or [Aliyun Console](https://www.aliyun.com/) if you are in China) and create your own [AccessKey](https://www.alibabacloud.com/help/doc-detail/29009.htm) which gives you the AccessKeyID (AKID) and AccessKeySecret (AKS). You can optionally create an Object Storage Service ([OSS](https://www.alibabacloud.com/help/doc-detail/31947.htm)) bucket for backups.
+2. Create a secret storing your AKID and AKS in Kubernetes. 
+
+ ```yaml 
+apiVersion: v1
+ kind: Secret
+ metadata:
+ name: my-oss-credentials
+ type: Opaque
+ data:
+ accessKeyID: <my-access-key-id>
+ accessKeySecret: <my-access-key-secret>
+ ``` 
+
+3. Create an `EtcdBackup` CR file `etcdbackup.yaml` which uses secret `my-oss-credentials` from the previous step. 
+```yaml
+apiVersion: etcd.database.coreos.com/v1beta2
+kind: EtcdBackup
+metadata:
+ name: etcd-cluster-with-oss-backup
+spec:
+ backupPolicy:
+ ...
+ etcdEndpoints:
+ - "http://example-etcd-cluster-client:2379"
+ storageType: OSS
+ oss:
+ endpoint: http://oss-cn-hangzhou.aliyuncs.com
+ ossSecret: my-oss-credentials
+ path: my-etcd-backups-bucket/etcd.backup
+``` 
+
+4. Apply yaml file to kubernetes cluster. 
+```sh
+kubectl apply -f etcdbackup.yaml
+```
+5. Check the `status` section of the `EtcdBackup` CR.
+```console
+$ kubectl get EtcdBackup etcd-cluster-with-oss-backup -o yaml
+apiVersion: etcd.database.coreos.com/v1beta2
+kind: EtcdBackup
+...
+spec:
+ oss:
+ ossSecret: my-oss-credentials
+ path: my-etcd-backups-bucket/etcd.backup
+ endpoint: http://oss-cn-hangzhou.aliyuncs.com
+ etcdEndpoints:
+ - http://example-etcd-cluster-client:2379
+ storageType: OSS
+status:
+ etcdRevision: 1
+ etcdVersion: 3.2.13
+ succeeded: true
+```
+
+6. We should see the backup files from Alibaba Cloud OSS Console.
+
+
+## Restore etcd based on data from OSS.
+
+Etcd restore operator is in charge of restoring etcd cluster from backup. If it is not deployed, please deploy by following command:
+
+```sh
+kubectl apply -f example/etcd-restore-operator/deployment.yaml
+```
+
+Now kill all the etcd pods to simulate a cluster failure:
+
+```sh
+kubectl delete pod -l app=etcd,etcd_cluster=example-etcd-cluster --force --grace-period=0
+```
+
+1. Create an EtcdRestore CR.
+```yaml
+apiVersion: "etcd.database.coreos.com/v1beta2"
+kind: "EtcdRestore"
+metadata:
+ # The restore CR name must be the same as spec.etcdCluster.name
+ name: example-etcd-cluster
+spec:
+ etcdCluster:
+ # The namespace is the same as this EtcdRestore CR
+ name: example-etcd-cluster
+ backupStorageType: OSS
+ oss:
+ # The format of the path must be: "<oss-bucket-name>/<path-to-backup-file>"
+ path: my-etcd-backups-bucket/etcd.backup
+ ossSecret: my-oss-credentials
+ endpoint: http://oss-cn-hangzhou.aliyuncs.com
+```
+
+2. Check the `status` section of the `EtcdRestore` CR. 
+```sh
+$ kubectl get etcdrestore example-etcd-cluster -o yaml
+apiVersion: etcd.database.coreos.com/v1beta2
+kind: EtcdRestore
+...
+spec:
+ oss:
+ ossSecret: my-oss-credentials
+ path: my-etcd-backups-bucket/etcd.backup
+ endpoint: http://oss-cn-hangzhou.aliyuncs.com
+ backupStorageType: OSS
+ etcdCluster:
+ name: example-etcd-cluster
+status:
+ succeeded: true
+```
+
+3. Verify the `EtcdCluster` CR and restored pods for the restored cluster. 
+```sh 
+$ kubectl get etcdcluster
+$ kubectl get pods -l app=etcd,etcd_cluster=example-etcd-cluster
+```

pkg/apis/etcd/v1beta2/backup_types.go

@@ -34,6 +34,11 @@ const (
  BackupStorageTypeGCS BackupStorageType = "GCS"
  GCPAccessToken = "access-token"
  GCPCredentialsJson = "credentials.json"
+
+ // Alibaba Cloud OSS related consts
+ BackupStorageTypeOSS BackupStorageType = "OSS"
+ AlibabaCloudSecretCredentialsAccessKeyID = "accessKeyID"
+ AlibabaCloudSecretCredentialsAccessKeySecret = "accessKeySecret"
 )
 
 type BackupStorageType string
@@ -90,6 +95,8 @@ type BackupSource struct {
  ABS *ABSBackupSource `json:"abs,omitempty"`
  // GCS defines the GCS backup source spec.
  GCS *GCSBackupSource `json:"gcs,omitempty"`
+ // OSS defines the OSS backup source spec.
+ OSS *OSSBackupSource `json:"oss,omitempty"`
 }
 
 // BackupPolicy defines backup policy.
@@ -169,3 +176,38 @@ type GCSBackupSource struct {
  // If omitted, client will use the default application credentials.
  GCPSecret string `json:"gcpSecret,omitempty"`
 }
+
+// OSSBackupSource provides the spec how to store backups on OSS.
+type OSSBackupSource struct {
+ // Path is the full abs path where the backup is saved.
+ // The format of the path must be: "<oss-bucket-name>/<path-to-backup-file>"
+ // e.g: "mybucket/etcd.backup"
+ Path string `json:"path"`
+
+ // The name of the secret object that stores the credential which will be used
+ // to access Alibaba Cloud OSS.
+ //
+ // The secret must contain the following keys/fields:
+ // accessKeyID
+ // accessKeySecret
+ //
+ // The format of secret:
+ //
+ // apiVersion: v1
+ // kind: Secret
+ // metadata:
+ // name: <my-credential-name>
+ // type: Opaque
+ // data:
+ // accessKeyID: <base64 of my-access-key-id>
+ // accessKeySecret: <base64 of my-access-key-secret>
+ //
+ OSSSecret string `json:"ossSecret"`
+
+ // Endpoint is the OSS service endpoint on alibaba cloud, defaults to
+ // "http://oss-cn-hangzhou.aliyuncs.com".
+ //
+ // Details about regions and endpoints, see:
+ // https://www.alibabacloud.com/help/doc-detail/31837.htm
+ Endpoint string `json:"endpoint,omitempty"`
+}

pkg/apis/etcd/v1beta2/restore_types.go

@@ -69,6 +69,9 @@ type RestoreSource struct {
 
  // GCS tells where on GCS the backup is saved and how to fetch the backup.
  GCS *GCSRestoreSource `json:"gcs,omitempty"`
+
+ // OSS tells where on OSS the backup is saved and how to fetch the backup.
+ OSS *OSSRestoreSource `json:"oss,omitempty"`
 }
 
 type S3RestoreSource struct {
@@ -120,6 +123,40 @@ type GCSRestoreSource struct {
  GCPSecret string `json:"gcpSecret,omitempty"`
 }
 
+type OSSRestoreSource struct {
+ // Path is the full abs path where the backup is saved.
+ // The format of the path must be: "<oss-bucket-name>/<path-to-backup-file>"
+ // e.g: "myossbucket/etcd.backup"
+ Path string `json:"path"`
+
+ // The name of the secret object that stores the credential which will be used
+ // to access Alibaba Cloud OSS.
+ //
+ // The secret must contain the following keys/fields:
+ // accessKeyID
+ // accessKeySecret
+ //
+ // The format of secret:
+ //
+ // apiVersion: v1
+ // kind: Secret
+ // metadata:
+ // name: <my-credential-name>
+ // type: Opaque
+ // data:
+ // accessKeyID: <base64 of my-access-key-id>
+ // accessKeySecret: <base64 of my-access-key-secret>
+ //
+ OSSSecret string `json:"ossSecret"`
+
+ // Endpoint is the OSS service endpoint on alibaba cloud, defaults to
+ // "http://oss-cn-hangzhou.aliyuncs.com".
+ //
+ // Details about regions and endpoints, see:
+ // https://www.alibabacloud.com/help/doc-detail/31837.htm
+ Endpoint string `json:"endpoint,omitempty"`
+}
+
 // RestoreStatus reports the status of this restore operation.
 type RestoreStatus struct {
  // Succeeded indicates if the backup has Succeeded.