From a3ff7e03bc599cf47e2152648adccaac1c5fc2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=81nis=20Bebr=C4=ABtis?= Date: Tue, 27 Aug 2024 11:50:19 +0300 Subject: [PATCH] Provisioner support --- CHANGELOG.txt | 18 ++- README.md | 18 ++- VERSION | 2 +- cmd/csi-rclone-plugin/main.go | 1 + .../kubernetes/1.13/csi-controller-rbac.yaml | 66 -------- .../1.13/csi-nodeplugin-rclone.yaml | 82 ---------- .../1.13/csi-rclone-storageclass.yaml | 6 - .../1.19/_csi-rclone-namespace.yaml | 4 - .../1.19/csi-controller-rclone.yaml | 54 ------- .../kubernetes/1.19/csi-nodeplugin-rbac.yaml | 40 ----- .../1.19/csi-rclone-storageclass.yaml | 5 - .../{1.13 => 1.20}/_csi-rclone-namespace.yaml | 0 .../{1.19 => 1.20}/csi-controller-rbac.yaml | 22 ++- .../{1.13 => 1.20}/csi-controller-rclone.yaml | 28 ++-- .../kubernetes/{1.19 => 1.20}/csi-driver.yaml | 2 +- .../{1.13 => 1.20}/csi-nodeplugin-rbac.yaml | 0 .../{1.19 => 1.20}/csi-nodeplugin-rclone.yaml | 5 +- .../1.20/csi-rclone-storageclass.yaml | 8 + example/kubernetes/nginx-pvc-example.yaml | 49 ++++++ go.mod | 3 +- go.sum | 1 + pkg/rclone/controller.go | 141 ++++++++++++++++++ pkg/rclone/driver.go | 25 +++- pkg/rclone/nodeserver.go | 30 ++-- 24 files changed, 302 insertions(+), 308 deletions(-) delete mode 100644 deploy/kubernetes/1.13/csi-controller-rbac.yaml delete mode 100644 deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml delete mode 100644 deploy/kubernetes/1.13/csi-rclone-storageclass.yaml delete mode 100644 deploy/kubernetes/1.19/_csi-rclone-namespace.yaml delete mode 100644 deploy/kubernetes/1.19/csi-controller-rclone.yaml delete mode 100644 deploy/kubernetes/1.19/csi-nodeplugin-rbac.yaml delete mode 100644 deploy/kubernetes/1.19/csi-rclone-storageclass.yaml rename deploy/kubernetes/{1.13 => 1.20}/_csi-rclone-namespace.yaml (100%) rename deploy/kubernetes/{1.19 => 1.20}/csi-controller-rbac.yaml (61%) rename deploy/kubernetes/{1.13 => 1.20}/csi-controller-rclone.yaml (71%) rename deploy/kubernetes/{1.19 => 1.20}/csi-driver.yaml (75%) rename deploy/kubernetes/{1.13 => 1.20}/csi-nodeplugin-rbac.yaml (100%) rename deploy/kubernetes/{1.19 => 1.20}/csi-nodeplugin-rclone.yaml (96%) create mode 100644 deploy/kubernetes/1.20/csi-rclone-storageclass.yaml create mode 100644 example/kubernetes/nginx-pvc-example.yaml create mode 100644 pkg/rclone/controller.go diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 00f2aef..330364c 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,21 @@ # CHANGELOG +3.0.0: + - Dropping support for kubernetes versions < 1.20 due to [external provisioner](https://github.com/kubernetes-csi/external-provisioner?tab=readme-ov-file#compatibility)+ compatibility requirement. + - PersistentVolume provisioner support. Creation of PersistentVolume and using it in PersistentVolumeClaim via selector is still supported, + no migration is required. + If PersistentVolume was setting `remotePathSuffix`, it can be added to PVC annotations as `csi-rclone/storage-path` value now (namespace can be ommited if `pathPattern` includes it - `${.PVC.namespace}/${.PVC.annotations.csi-rclone/storage-path}`). + If PersistentVolume was setting umask parameter, it's possible to set it in PVC annotations as `csi-rclone/umask` value now. + - cluster-driver-registrar is not required anymore since deployment resources declare CSIDriver object and registrar is [deprecated](https://kubernetes-csi.github.io/docs/cluster-driver-registrar.html#deprecated). + +2.0.0: + - rclone version v1.66.0 + - Custom rclone build (directory markers) removed since it's available in the official binary now + - Separate cache paths for each mount process + cache removal on unmount + - Remote control API endpoint for each mount process, this allows transfer state and vfs upload buffer queue monitoring + - Delay rclone process shutdown until upload queue is empty (There's an additional timeout of 1 hour. That should be enough even for bigger files) + - Graceful failover for plugin preStart + 1.3.0: - Container init changed to tini - - rclone plugin version v1.59.2 + - rclone version v1.59.2 \ No newline at end of file diff --git a/README.md b/README.md index 441ed62..8f347c3 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,14 @@ This project implements Container Storage Interface (CSI) plugin that allows usi ## Kubernetes cluster compatability Works (tested): -- `deploy/kubernetes/1.19`: K8S>= 1.19.x (due to storage.k8s.io/v1 CSIDriver API) -- `deploy/kubernetes/1.13`: K8S 1.13.x - 1.21.x (storage.k8s.io/v1beta1 CSIDriver API) - -Does not work: -- v1.12.7-gke.10, driver name csi-rclone not found in the list of registered CSI drivers +- `deploy/kubernetes/1.20`: K8S>= 1.20.x External provisioner requires kubernetes [1.20](https://github.com/kubernetes-csi/external-provisioner?tab=readme-ov-file#compatibility)+. +- Older driver versions (before v3.0.0) support kubernetes 1.13-1.19, but are not maintained. ## Installing CSI driver to kubernetes cluster -TLDR: ` kubectl apply -f deploy/kubernetes/1.19` (or `deploy/kubernetes/1.13` for older version) +TLDR: `kubectl apply -f deploy/kubernetes/1.20` 1. Set up storage backend. You can use [Minio](https://min.io/), Amazon S3 compatible cloud storage service. -i.e. +i.e (heads up - minio setup example is severly outdated). ``` helm upgrade --install --create-namespace --namespace minio minio minio/minio --version 6.0.5 --set resources.requests.memory=512Mi --set secretKey=SECRET_ACCESS_KEY --set accessKey=ACCESS_KEY_ID ``` @@ -91,6 +88,13 @@ Deploy example definition > `kubectl apply -f example/kubernetes/nginx-example.yaml` +## PersistentVolumeClaim annotations + +- `csi-rclone/umask` - `umask` parameter for `rclone mount`. +- [if configured in storageclass `parameters.pathPattern`] `csi-rclone/storage-path` - Secret name that contains rclone configuration. + +Provisioning of other parameters is currently unsupported, create PersistentVolume resource with `volumeAttributes` to define them. + ## Building plugin and creating image Current code is referencing projects repository on github.com. If you fork the repository, you have to change go includes in several places (use search and replace). diff --git a/VERSION b/VERSION index 46b105a..1c5ae58 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.0 +v3.0.0 \ No newline at end of file diff --git a/cmd/csi-rclone-plugin/main.go b/cmd/csi-rclone-plugin/main.go index edd8f31..bd58a24 100644 --- a/cmd/csi-rclone-plugin/main.go +++ b/cmd/csi-rclone-plugin/main.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "os" + "github.com/spf13/cobra" "github.com/wunderio/csi-rclone/pkg/rclone" ) diff --git a/deploy/kubernetes/1.13/csi-controller-rbac.yaml b/deploy/kubernetes/1.13/csi-controller-rbac.yaml deleted file mode 100644 index ba9b642..0000000 --- a/deploy/kubernetes/1.13/csi-controller-rbac.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# This YAML file contains RBAC API objects that are necessary to run external -# CSI attacher for rclone adapter - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-controller-rclone - namespace: csi-rclone ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-controller-rclone -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-role-rclone -subjects: - - kind: ServiceAccount - name: csi-controller-rclone - namespace: csi-rclone -roleRef: - kind: ClusterRole - name: external-controller-rclone - apiGroup: rbac.authorization.k8s.io ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-cluster-driver-registrar-role -rules: - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csidrivers"] - verbs: ["create", "delete"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-cluster-driver-registrar-binding -subjects: - - kind: ServiceAccount - name: csi-controller-rclone - namespace: csi-rclone -roleRef: - kind: ClusterRole - name: csi-cluster-driver-registrar-role - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml b/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml deleted file mode 100644 index a6ff091..0000000 --- a/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# This YAML file contains driver-registrar & csi driver nodeplugin API objects -# that are necessary to run CSI nodeplugin for rclone -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: csi-nodeplugin-rclone - namespace: csi-rclone -spec: - selector: - matchLabels: - app: csi-nodeplugin-rclone - template: - metadata: - labels: - app: csi-nodeplugin-rclone - spec: - serviceAccountName: csi-nodeplugin-rclone - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: node-driver-registrar - image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/csi-rclone /registration/csi-rclone-reg.sock"] - args: - - --v=5 - - --csi-address=/plugin/csi.sock - - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-rclone/csi.sock - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /plugin - - name: registration-dir - mountPath: /registration - - name: rclone - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: wunderio/csi-rclone:v2.0.0 - args: - - "/bin/csi-rclone-plugin" - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://plugin/csi.sock - imagePullPolicy: "Always" - lifecycle: - postStart: - exec: - command: ["/bin/sh", "-c", "mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') || true ; done"] - volumeMounts: - - name: plugin-dir - mountPath: /plugin - - name: pods-mount-dir - mountPath: /var/lib/kubelet/pods - mountPropagation: "Bidirectional" - volumes: - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/csi-rclone - type: DirectoryOrCreate - - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet/pods - type: Directory - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: DirectoryOrCreate - name: registration-dir diff --git a/deploy/kubernetes/1.13/csi-rclone-storageclass.yaml b/deploy/kubernetes/1.13/csi-rclone-storageclass.yaml deleted file mode 100644 index 5a32d4c..0000000 --- a/deploy/kubernetes/1.13/csi-rclone-storageclass.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rclone - namespace: csi-rclone -provisioner: kubernetes.io/no-provisioner diff --git a/deploy/kubernetes/1.19/_csi-rclone-namespace.yaml b/deploy/kubernetes/1.19/_csi-rclone-namespace.yaml deleted file mode 100644 index ac9b8a5..0000000 --- a/deploy/kubernetes/1.19/_csi-rclone-namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: csi-rclone \ No newline at end of file diff --git a/deploy/kubernetes/1.19/csi-controller-rclone.yaml b/deploy/kubernetes/1.19/csi-controller-rclone.yaml deleted file mode 100644 index ba50af9..0000000 --- a/deploy/kubernetes/1.19/csi-controller-rclone.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# This YAML file contains attacher & csi driver API objects that are necessary -# to run external CSI attacher for rclone - -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-controller-rclone - namespace: csi-rclone -spec: - serviceName: "csi-controller-rclone" - replicas: 1 - selector: - matchLabels: - app: csi-controller-rclone - template: - metadata: - labels: - app: csi-controller-rclone - spec: - serviceAccountName: csi-controller-rclone - containers: - - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--leader-election" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /csi - - name: rclone - image: wunderio/csi-rclone:v2.0.0 - args : - - "/bin/csi-rclone-plugin" - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://plugin/csi.sock - imagePullPolicy: "Always" - volumeMounts: - - name: socket-dir - mountPath: /plugin - volumes: - - name: socket-dir - emptyDir: {} diff --git a/deploy/kubernetes/1.19/csi-nodeplugin-rbac.yaml b/deploy/kubernetes/1.19/csi-nodeplugin-rbac.yaml deleted file mode 100644 index 2e43700..0000000 --- a/deploy/kubernetes/1.19/csi-nodeplugin-rbac.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# This YAML defines all API objects to create RBAC roles for CSI node plugin -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-nodeplugin-rclone - namespace: csi-rclone ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-rclone -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["secrets","secret"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-nodeplugin-rclone -subjects: - - kind: ServiceAccount - name: csi-nodeplugin-rclone - namespace: csi-rclone -roleRef: - kind: ClusterRole - name: csi-nodeplugin-rclone - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/1.19/csi-rclone-storageclass.yaml b/deploy/kubernetes/1.19/csi-rclone-storageclass.yaml deleted file mode 100644 index 4bbafd7..0000000 --- a/deploy/kubernetes/1.19/csi-rclone-storageclass.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: rclone -provisioner: kubernetes.io/no-provisioner diff --git a/deploy/kubernetes/1.13/_csi-rclone-namespace.yaml b/deploy/kubernetes/1.20/_csi-rclone-namespace.yaml similarity index 100% rename from deploy/kubernetes/1.13/_csi-rclone-namespace.yaml rename to deploy/kubernetes/1.20/_csi-rclone-namespace.yaml diff --git a/deploy/kubernetes/1.19/csi-controller-rbac.yaml b/deploy/kubernetes/1.20/csi-controller-rbac.yaml similarity index 61% rename from deploy/kubernetes/1.19/csi-controller-rbac.yaml rename to deploy/kubernetes/1.20/csi-controller-rbac.yaml index 5086a31..0b43c84 100644 --- a/deploy/kubernetes/1.19/csi-controller-rbac.yaml +++ b/deploy/kubernetes/1.20/csi-controller-rbac.yaml @@ -14,6 +14,9 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch", "update", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["nodes"] @@ -23,7 +26,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "create", "delete"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] @@ -32,7 +35,22 @@ rules: verbs: ["get", "create", "update"] - apiGroups: [""] resources: ["events"] - verbs: ["create"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/deploy/kubernetes/1.13/csi-controller-rclone.yaml b/deploy/kubernetes/1.20/csi-controller-rclone.yaml similarity index 71% rename from deploy/kubernetes/1.13/csi-controller-rclone.yaml rename to deploy/kubernetes/1.20/csi-controller-rclone.yaml index 8627168..39f623d 100644 --- a/deploy/kubernetes/1.13/csi-controller-rclone.yaml +++ b/deploy/kubernetes/1.20/csi-controller-rclone.yaml @@ -19,36 +19,40 @@ spec: spec: serviceAccountName: csi-controller-rclone containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v1.1.1 + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v5.0.2 args: - - "--v=5" - "--csi-address=$(ADDRESS)" + - "--extra-create-metadata" + # - "--leader-election" + - "--v=1" env: - name: ADDRESS - value: /csi/csi.sock + value: /plugin/csi.sock imagePullPolicy: "Always" volumeMounts: - name: socket-dir - mountPath: /csi - - name: csi-cluster-driver-registrar - image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1 + mountPath: /plugin + - name: csi-attacher + image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 args: - - "--v=5" - - "--pod-info-mount-version=\"v1\"" - "--csi-address=$(ADDRESS)" + - "--v=1" + # - "--leader-election" env: - name: ADDRESS - value: /csi/csi.sock + value: /plugin/csi.sock + imagePullPolicy: "Always" volumeMounts: - name: socket-dir - mountPath: /csi + mountPath: /plugin - name: rclone - image: wunderio/csi-rclone:v2.0.0 + image: wunderio/csi-rclone:v3.0.0 args : - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" + - "--v=1" env: - name: NODE_ID valueFrom: diff --git a/deploy/kubernetes/1.19/csi-driver.yaml b/deploy/kubernetes/1.20/csi-driver.yaml similarity index 75% rename from deploy/kubernetes/1.19/csi-driver.yaml rename to deploy/kubernetes/1.20/csi-driver.yaml index d76b42e..39d2317 100644 --- a/deploy/kubernetes/1.19/csi-driver.yaml +++ b/deploy/kubernetes/1.20/csi-driver.yaml @@ -5,4 +5,4 @@ metadata: name: csi-rclone spec: attachRequired: true - podInfoOnMount: false # are we sure about this? + podInfoOnMount: true \ No newline at end of file diff --git a/deploy/kubernetes/1.13/csi-nodeplugin-rbac.yaml b/deploy/kubernetes/1.20/csi-nodeplugin-rbac.yaml similarity index 100% rename from deploy/kubernetes/1.13/csi-nodeplugin-rbac.yaml rename to deploy/kubernetes/1.20/csi-nodeplugin-rbac.yaml diff --git a/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml b/deploy/kubernetes/1.20/csi-nodeplugin-rclone.yaml similarity index 96% rename from deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml rename to deploy/kubernetes/1.20/csi-nodeplugin-rclone.yaml index caf645c..dd156a7 100644 --- a/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml +++ b/deploy/kubernetes/1.20/csi-nodeplugin-rclone.yaml @@ -25,7 +25,7 @@ spec: exec: command: ["/bin/sh", "-c", "rm -rf /registration/csi-rclone /registration/csi-rclone-reg.sock"] args: - - --v=5 + - --v=1 - --csi-address=/plugin/csi.sock - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-rclone/csi.sock env: @@ -44,11 +44,12 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: wunderio/csi-rclone:v2.0.0 + image: wunderio/csi-rclone:v3.0.0 args: - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" + - "--v=1" env: - name: NODE_ID valueFrom: diff --git a/deploy/kubernetes/1.20/csi-rclone-storageclass.yaml b/deploy/kubernetes/1.20/csi-rclone-storageclass.yaml new file mode 100644 index 0000000..8b0a0f3 --- /dev/null +++ b/deploy/kubernetes/1.20/csi-rclone-storageclass.yaml @@ -0,0 +1,8 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rclone +# You will need to delete storageclass to update this field +provisioner: csi-rclone +# parameters: +# pathPattern: "${.PVC.namespace}/${.PVC.annotations.csi-rclone/storage-path}" \ No newline at end of file diff --git a/example/kubernetes/nginx-pvc-example.yaml b/example/kubernetes/nginx-pvc-example.yaml new file mode 100644 index 0000000..0d20ced --- /dev/null +++ b/example/kubernetes/nginx-pvc-example.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: data-rclone-example + annotations: + csi-rclone/storage-path: nginx + csi-rclone/umask: "022" +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: rclone +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-example + labels: + run: nginx-example +spec: + containers: + - image: nginx + imagePullPolicy: Always + name: nginx-example + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /usr/share/nginx/html + name: data-rclone-example + volumes: + - name: data-rclone-example + persistentVolumeClaim: + claimName: data-rclone-example +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-example + labels: + run: nginx-example +spec: + ports: + - port: 80 + protocol: TCP + selector: + run: nginx-example diff --git a/go.mod b/go.mod index f1a49de..e95e175 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/gogo/protobuf v1.2.1 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect @@ -44,7 +45,7 @@ require ( k8s.io/client-go v10.0.0+incompatible k8s.io/cloud-provider v0.0.0-20190223141949-e954a34baf43 // indirect k8s.io/csi-api v0.0.0-20190223140843-b4e64dae0b19 // indirect - k8s.io/klog v0.2.0 + k8s.io/klog v0.2.0 // indirect k8s.io/kube-openapi v0.0.0-20190222203931-aa8624f5a2df // indirect k8s.io/kubernetes v1.13.2 k8s.io/utils v0.0.0-20190221042446-c2654d5206da // indirect diff --git a/go.sum b/go.sum index 3a88191..9dd1b17 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,7 @@ github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BU github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= diff --git a/pkg/rclone/controller.go b/pkg/rclone/controller.go new file mode 100644 index 0000000..7fa7fc4 --- /dev/null +++ b/pkg/rclone/controller.go @@ -0,0 +1,141 @@ +package rclone + +import ( + "regexp" + "strings" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type controllerServer struct { + *csicommon.DefaultControllerServer +} + +type pvcMetadata struct { + data map[string]string + labels map[string]string + annotations map[string]string +} + +// source: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/blob/master/cmd/nfs-subdir-external-provisioner/provisioner.go +var pattern = regexp.MustCompile(`\${\.PVC\.((labels|annotations)\.(.*?)|.*?)}`) + +// source: https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/blob/master/cmd/nfs-subdir-external-provisioner/provisioner.go +func (meta *pvcMetadata) stringParser(str string) string { + result := pattern.FindAllStringSubmatch(str, -1) + for _, r := range result { + switch r[2] { + case "labels": + str = strings.ReplaceAll(str, r[0], meta.labels[r[3]]) + case "annotations": + str = strings.ReplaceAll(str, r[0], meta.annotations[r[3]]) + default: + str = strings.ReplaceAll(str, r[0], meta.data[r[1]]) + } + } + + return str +} + +func (cs *controllerServer) getPVC(name, namespace string) (*v1.PersistentVolumeClaim, error) { + clientset, e := GetK8sClient() + if e != nil { + return nil, status.Errorf(codes.Internal, "can not create kubernetes client: %s", e) + } + + // Get the PVC + pvc, err := clientset.CoreV1().PersistentVolumeClaims(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + glog.Errorf("Failed to get PVC: %v", err) + return nil, err + } + + return pvc, nil +} + +func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + // Parse the request to get the volume name, size, and parameters. + volumeName := req.GetName() + capacityBytes := req.GetCapacityRange().GetRequiredBytes() + + // Extract parameters from the request + parameters := req.GetParameters() + + volumeContext := map[string]string{} + + pvcName := "" + pvcNamespace := "" + + // parameter provided by external-provisioner (csi-provisioner) + if val, ok := parameters["csi.storage.k8s.io/pvc/name"]; ok { + pvcName = val + } + + // parameter provided by external-provisioner (csi-provisioner) + if val, ok := parameters["csi.storage.k8s.io/pvc/namespace"]; ok { + pvcNamespace = val + } + + // If PVC name is provided, load the PVC definition + if pvcName != "" { + + pvc, err := cs.getPVC(pvcName, pvcNamespace) + if err != nil { + glog.Errorf("Failed to get PVC %s in namespace %s: %v", pvcName, pvcNamespace, err) + return nil, err + } + + // Extract PVC metadata + metadata := &pvcMetadata{ + data: map[string]string{ + "name": pvcName, + "namespace": pvcNamespace, + }, + labels: pvc.Labels, + annotations: pvc.Annotations, + } + + if pathPattern, ok := parameters["pathPattern"]; ok { + if pathPattern != "" { + remotePathSuffix := metadata.stringParser(pathPattern) + if remotePathSuffix != "" { + if !strings.HasPrefix(remotePathSuffix, "/") { + remotePathSuffix = "/" + remotePathSuffix + } + volumeContext["remotePathSuffix"] = remotePathSuffix + } + } + } + + // if Annotation starts with "csi-rclone/", extract the key and value from the annotation + for key, value := range metadata.annotations { + if strings.HasPrefix(key, "csi-rclone/") { + key = strings.TrimPrefix(key, "csi-rclone/") + + // Only allow some keys (umask, uid) to be passed to the volume context to avoid security issues + if key == "umask" { + volumeContext[key] = value + } + } + } + } + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volumeName, + CapacityBytes: capacityBytes, + VolumeContext: volumeContext, + }, + }, nil +} + +func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + return &csi.DeleteVolumeResponse{}, nil +} diff --git a/pkg/rclone/driver.go b/pkg/rclone/driver.go index 5f3c945..4f6c790 100644 --- a/pkg/rclone/driver.go +++ b/pkg/rclone/driver.go @@ -2,17 +2,16 @@ package rclone import ( "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" - "k8s.io/klog" ) type Driver struct { csiDriver *csicommon.CSIDriver endpoint string - ns *nodeServer - cap []*csi.VolumeCapability_AccessMode - cscap []*csi.ControllerServiceCapability + ns *nodeServer + cs *controllerServer } var ( @@ -21,7 +20,7 @@ var ( ) func NewDriver(nodeID, endpoint string) *Driver { - klog.Infof("Starting new %s driver in version %s", DriverName, DriverVersion) + glog.Infof("Starting new %s driver in version %s", DriverName, DriverVersion) d := &Driver{} @@ -29,7 +28,10 @@ func NewDriver(nodeID, endpoint string) *Driver { d.csiDriver = csicommon.NewCSIDriver(DriverName, DriverVersion, nodeID) d.csiDriver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) - d.csiDriver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_UNKNOWN}) + d.csiDriver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME}) + + d.cs = NewControllerServer(d) + d.ns = NewNodeServer(d) return d } @@ -40,11 +42,18 @@ func NewNodeServer(d *Driver) *nodeServer { } } +func NewControllerServer(d *Driver) *controllerServer { + return &controllerServer{ + DefaultControllerServer: csicommon.NewDefaultControllerServer(d.csiDriver), + } +} + func (d *Driver) Run() { s := csicommon.NewNonBlockingGRPCServer() s.Start(d.endpoint, csicommon.NewDefaultIdentityServer(d.csiDriver), - csicommon.NewDefaultControllerServer(d.csiDriver), - NewNodeServer(d)) + d.cs, + d.ns, + ) s.Wait() } diff --git a/pkg/rclone/nodeserver.go b/pkg/rclone/nodeserver.go index d01b5e7..09baefe 100644 --- a/pkg/rclone/nodeserver.go +++ b/pkg/rclone/nodeserver.go @@ -14,9 +14,9 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -58,7 +58,7 @@ func (ns *nodeServer) deleteMountContext(targetPath string) { } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { - klog.Infof("NodePublishVolume: called with args %+v", *req) + glog.V(4).Infof("NodePublishVolume: called with args %+v", *req) targetPath := req.GetTargetPath() @@ -77,11 +77,11 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if !notMnt { // testing original mount point, make sure the mount link is valid if _, err := ioutil.ReadDir(targetPath); err == nil { - klog.Infof("already mounted to target %s", targetPath) + glog.V(4).Infof("already mounted to target %s", targetPath) return &csi.NodePublishVolumeResponse{}, nil } // todo: mount link is invalid, now unmount and remount later (built-in functionality) - klog.Warningf("ReadDir %s failed with %v, unmount this directory", targetPath, err) + glog.Warningf("ReadDir %s failed with %v, unmount this directory", targetPath, err) ns.mounter = &mount.SafeFormatAndMount{ Interface: mount.New(""), @@ -89,7 +89,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } if err := ns.mounter.Unmount(targetPath); err != nil { - klog.Errorf("Unmount directory %s failed with %v", targetPath, err) + glog.Errorf("Unmount directory %s failed with %v", targetPath, err) return nil, err } } @@ -104,7 +104,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext(), secret) if e != nil { - klog.Warningf("storage parameter error: %s", e) + glog.Warningf("storage parameter error: %s", e) return nil, e } @@ -140,7 +140,7 @@ func extractFlags(volumeContext map[string]string, secret *v1.Secret) (string, s flags[k] = string(v) } } else { - klog.Infof("No csi-rclone connection defaults secret found.") + glog.V(4).Infof("No csi-rclone connection defaults secret found.") } if len(volumeContext) > 0 { @@ -289,30 +289,28 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } if notMnt && !mount.IsCorruptedMnt(err) { - klog.Infof("Volume not mounted") + glog.V(4).Infof("Volume not mounted") } else { err = util.UnmountPath(req.GetTargetPath(), m) if err != nil { - klog.Infof("Error while unmounting path: %s", err) + glog.V(4).Infof("Error while unmounting path: %s", err) // This will exit and fail the NodeUnpublishVolume making it to retry unmount on the next api schedule trigger. // Since we mount the volume with allow-non-empty now, we could skip this one too. return nil, status.Error(codes.Internal, err.Error()) } - klog.Infof("Volume %s unmounted successfully", req.VolumeId) + glog.V(4).Infof("Volume %s unmounted successfully", req.VolumeId) } return &csi.NodeUnpublishVolumeResponse{}, nil } func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - klog.Infof("NodeUnstageVolume: called with args %+v", *req) return &csi.NodeUnstageVolumeResponse{}, nil } func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - klog.Infof("NodeStageVolume: called with args %+v", *req) return &csi.NodeStageVolumeResponse{}, nil } @@ -342,7 +340,7 @@ func getSecret(secretName string) (*v1.Secret, error) { return nil, status.Errorf(codes.Internal, "can't get current namespace, error %s", secretName, err) } - klog.Infof("Loading csi-rclone connection defaults from secret %s/%s", namespace, secretName) + glog.V(4).Infof("Loading csi-rclone connection defaults from secret %s/%s", namespace, secretName) secret, e := clientset.CoreV1(). Secrets(namespace). @@ -394,7 +392,7 @@ func Mount(remote string, remotePath string, targetPath string, configData strin if strings.Contains(configData, "["+remote+"]") { remoteWithPath = fmt.Sprintf("%s:%s", remote, remotePath) - klog.Infof("remote %s found in configData, remoteWithPath set to %s", remote, remoteWithPath) + glog.V(4).Infof("remote %s found in configData, remoteWithPath set to %s", remote, remoteWithPath) } // Find a free port for rclone rc @@ -464,8 +462,8 @@ func Mount(remote string, remotePath string, targetPath string, configData strin return 0, err } - klog.Infof("executing mount command cmd=%s, remote=%s, targetpath=%s", mountCmd, remoteWithPath, targetPath) - klog.Infof("mountArgs: %v", mountArgs) + glog.V(4).Infof("executing mount command cmd=%s, remote=%s, targetpath=%s", mountCmd, remoteWithPath, targetPath) + glog.V(4).Infof("mountArgs: %v", mountArgs) cmd := exec.Command(mountCmd, mountArgs...) cmd.Env = env