diff --git a/Dockerfile.dm b/Dockerfile.dm deleted file mode 100644 index 54b1d0c..0000000 --- a/Dockerfile.dm +++ /dev/null @@ -1,23 +0,0 @@ -#### -FROM golang:alpine AS builder -RUN apk update && apk add --no-cache git make bash -WORKDIR $GOPATH/src/csi-rclone-nodeplugin -COPY . . -RUN make plugin-dm - -#### -FROM alpine:3.16 -RUN apk add --no-cache ca-certificates bash fuse3 curl unzip tini - -# RUN curl https://rclone.org/install.sh | bash - -# Use pre-compiled version (with cirectory marker patch) -# https://github.com/rclone/rclone/pull/5323 -COPY ./install-dm.sh /tmp -COPY ./rclone-build /tmp/rclone-build -RUN /tmp/install-dm.sh - -COPY --from=builder /go/src/csi-rclone-nodeplugin/_output/csi-rclone-plugin-dm /bin/csi-rclone-plugin - -ENTRYPOINT [ "/sbin/tini", "--"] -CMD ["/bin/csi-rclone-plugin"] diff --git a/Makefile b/Makefile index 69d7028..f0fa71a 100644 --- a/Makefile +++ b/Makefile @@ -20,28 +20,17 @@ IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION) .PHONY: all rclone-plugin clean rclone-container all: plugin container push -dm: plugin-dm container-dm push-dm plugin: go mod download CGO_ENABLED=0 GOOS=linux go build -a -gcflags=-trimpath=$(go env GOPATH) -asmflags=-trimpath=$(go env GOPATH) -ldflags '-X github.com/wunderio/csi-rclone/pkg/rclone.DriverVersion=$(VERSION) -extldflags "-static"' -o _output/csi-rclone-plugin ./cmd/csi-rclone-plugin -plugin-dm: - go mod download - CGO_ENABLED=0 GOOS=linux go build -a -gcflags=-trimpath=$(go env GOPATH) -asmflags=-trimpath=$(go env GOPATH) -ldflags '-X github.com/wunderio/csi-rclone/pkg/rclone.DriverVersion=$(VERSION)-dm -extldflags "-static"' -o _output/csi-rclone-plugin-dm ./cmd/csi-rclone-plugin - container: docker build -t $(IMAGE_TAG) -f ./cmd/csi-rclone-plugin/Dockerfile . -container-dm: - docker build -t $(IMAGE_TAG)-dm -f ./cmd/csi-rclone-plugin/Dockerfile.dm . - push: docker push $(IMAGE_TAG) -push-dm: - docker push $(IMAGE_TAG)-dm - clean: go clean -r -x -rm -rf _output diff --git a/VERSION b/VERSION index ea7786a..46b105a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.3.3 +v2.0.0 diff --git a/cmd/csi-rclone-plugin/Dockerfile b/cmd/csi-rclone-plugin/Dockerfile index 237f2d7..faddee9 100644 --- a/cmd/csi-rclone-plugin/Dockerfile +++ b/cmd/csi-rclone-plugin/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16 -RUN apk add --no-cache ca-certificates bash fuse curl unzip tini +RUN apk add --no-cache ca-certificates bash fuse fuse3 curl unzip tini RUN curl https://rclone.org/install.sh | bash diff --git a/cmd/csi-rclone-plugin/Dockerfile.dm b/cmd/csi-rclone-plugin/Dockerfile.dm deleted file mode 100644 index 41c7951..0000000 --- a/cmd/csi-rclone-plugin/Dockerfile.dm +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:3.16 -RUN apk add --no-cache ca-certificates bash fuse curl unzip tini - -# RUN curl https://rclone.org/install.sh | bash - -# Use pre-compiled version (with cirectory marker patch) -# https://github.com/rclone/rclone/pull/5323 -COPY ./install-dm.sh /tmp -COPY ./rclone-build /tmp/rclone-build -RUN /tmp/install-dm.sh - -COPY ./_output/csi-rclone-plugin-dm /bin/csi-rclone-plugin - -ENTRYPOINT [ "/sbin/tini", "--"] -CMD ["/bin/csi-rclone-plugin"] \ No newline at end of file diff --git a/deploy/kubernetes/1.13/csi-controller-rclone.yaml b/deploy/kubernetes/1.13/csi-controller-rclone.yaml index fa17a7a..8627168 100644 --- a/deploy/kubernetes/1.13/csi-controller-rclone.yaml +++ b/deploy/kubernetes/1.13/csi-controller-rclone.yaml @@ -44,7 +44,7 @@ spec: - name: socket-dir mountPath: /csi - name: rclone - image: wunderio/csi-rclone:v1.3.1 + image: wunderio/csi-rclone:v2.0.0 args : - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" diff --git a/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml b/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml index b4c83f0..a6ff091 100644 --- a/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml +++ b/deploy/kubernetes/1.13/csi-nodeplugin-rclone.yaml @@ -44,7 +44,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: wunderio/csi-rclone:v1.3.1 + image: wunderio/csi-rclone:v2.0.0 args: - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" @@ -60,7 +60,7 @@ spec: lifecycle: postStart: exec: - command: ["/bin/sh", "-c", "mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') ; done"] + command: ["/bin/sh", "-c", "mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') || true ; done"] volumeMounts: - name: plugin-dir mountPath: /plugin diff --git a/deploy/kubernetes/1.19/csi-controller-rclone.yaml b/deploy/kubernetes/1.19/csi-controller-rclone.yaml index a4ade09..ba50af9 100644 --- a/deploy/kubernetes/1.19/csi-controller-rclone.yaml +++ b/deploy/kubernetes/1.19/csi-controller-rclone.yaml @@ -33,7 +33,7 @@ spec: - name: socket-dir mountPath: /csi - name: rclone - image: wunderio/csi-rclone:v1.3.1 + image: wunderio/csi-rclone:v2.0.0 args : - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" diff --git a/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml b/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml index f2711f2..caf645c 100644 --- a/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml +++ b/deploy/kubernetes/1.19/csi-nodeplugin-rclone.yaml @@ -44,7 +44,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: wunderio/csi-rclone:v1.3.1 + image: wunderio/csi-rclone:v2.0.0 args: - "/bin/csi-rclone-plugin" - "--nodeid=$(NODE_ID)" @@ -60,7 +60,7 @@ spec: lifecycle: postStart: exec: - command: ["/bin/sh", "-c", "mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') ; done"] + command: ["/bin/sh", "-c", "mount -t fuse.rclone | while read -r mount; do umount $(echo $mount | awk '{print $3}') || true ; done"] volumeMounts: - name: plugin-dir mountPath: /plugin diff --git a/go.mod b/go.mod index 2da4654..f1a49de 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/gogo/protobuf v1.2.1 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect diff --git a/go.sum b/go.sum index 87a06e0..3a88191 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E= github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -80,7 +79,6 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -96,11 +94,9 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jY golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -124,7 +120,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/install-dm.sh b/install-dm.sh deleted file mode 100755 index a918e20..0000000 --- a/install-dm.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash - -# error codes -# 0 - exited without problems -# 1 - parameters not supported were used or some unexpected error occurred -# 2 - OS not supported by this script -# 3 - installed version of rclone is up to date -# 4 - supported unzip tools are not available - -set -e - -#when adding a tool to the list make sure to also add its corresponding command further in the script -unzip_tools_list=('unzip' '7z' 'busybox') - -usage() { echo "Usage: sudo -v ; sudo bash install-dm.sh" 1>&2; exit 1; } - -#check for beta flag -if [ -n "$1" ] && [ "$1" != "beta" ]; then - usage -fi - -#create tmp directory and move to it with macOS compatibility fallback -tmp_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'rclone-install.XXXXXXXXXX') -cd "$tmp_dir" - -#make sure unzip tool is available and choose one to work with -set +e -for tool in ${unzip_tools_list[*]}; do - trash=$(hash "$tool" 2>>errors) - if [ "$?" -eq 0 ]; then - unzip_tool="$tool" - break - fi -done -set -e - -# exit if no unzip tools available -if [ -z "$unzip_tool" ]; then - printf "\nNone of the supported tools for extracting zip archives (${unzip_tools_list[*]}) were found. " - printf "Please install one of them and try again.\n\n" - exit 4 -fi - -# Make sure we don't create a root owned .config/rclone directory #2127 -export XDG_CONFIG_HOME=config - -#detect the platform -OS="$(uname)" -case $OS in - Linux) - OS='linux' - ;; - FreeBSD) - OS='freebsd' - ;; - NetBSD) - OS='netbsd' - ;; - OpenBSD) - OS='openbsd' - ;; - Darwin) - OS='osx' - binTgtDir=/usr/local/bin - man1TgtDir=/usr/local/share/man/man1 - ;; - SunOS) - OS='solaris' - echo 'OS not supported' - exit 2 - ;; - *) - echo 'OS not supported' - exit 2 - ;; -esac - -OS_type="$(uname -m)" -case "$OS_type" in - x86_64|amd64) - OS_type='amd64' - ;; - i?86|x86) - OS_type='386' - ;; - aarch64|arm64) - OS_type='arm64' - ;; - arm*) - OS_type='arm' - ;; - *) - echo 'OS type not supported' - exit 2 - ;; -esac - - -#download and unzip -rclone_zip="/tmp/rclone-build/rclone-current-${OS}-${OS_type}.zip" - -unzip_dir="tmp_unzip_dir_for_rclone" -# there should be an entry in this switch for each element of unzip_tools_list -case "$unzip_tool" in - 'unzip') - unzip -a "$rclone_zip" -d "$unzip_dir" - ;; - '7z') - 7z x "$rclone_zip" "-o$unzip_dir" - ;; - 'busybox') - mkdir -p "$unzip_dir" - busybox unzip "$rclone_zip" -d "$unzip_dir" - ;; -esac - -cd $unzip_dir/* - -#mounting rclone to environment - -case "$OS" in - 'linux') - #binary - cp rclone /usr/bin/rclone.new - chmod 755 /usr/bin/rclone.new - chown root:root /usr/bin/rclone.new - mv /usr/bin/rclone.new /usr/bin/rclone - #manual - if ! [ -x "$(command -v mandb)" ]; then - echo 'mandb not found. The rclone man docs will not be installed.' - else - mkdir -p /usr/local/share/man/man1 - cp rclone.1 /usr/local/share/man/man1/ - mandb - fi - ;; - 'freebsd'|'openbsd'|'netbsd') - #binary - cp rclone /usr/bin/rclone.new - chown root:wheel /usr/bin/rclone.new - mv /usr/bin/rclone.new /usr/bin/rclone - #manual - mkdir -p /usr/local/man/man1 - cp rclone.1 /usr/local/man/man1/ - makewhatis - ;; - 'osx') - #binary - mkdir -m 0555 -p ${binTgtDir} - cp rclone ${binTgtDir}/rclone.new - mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone - chmod a=x ${binTgtDir}/rclone - #manual - mkdir -m 0555 -p ${man1TgtDir} - cp rclone.1 ${man1TgtDir} - chmod a=r ${man1TgtDir}/rclone.1 - ;; - *) - echo 'OS not supported' - exit 2 -esac - - -#update version variable post install -version=$(rclone --version 2>>errors | head -n 1) - -printf "\n${version} has successfully installed." -printf '\nNow run "rclone config" for setup. Check https://rclone.org/docs/ for more details.\n\n' -exit 0 - diff --git a/pkg/rclone/driver.go b/pkg/rclone/driver.go index 7799abf..5f3c945 100644 --- a/pkg/rclone/driver.go +++ b/pkg/rclone/driver.go @@ -6,7 +6,7 @@ import ( "k8s.io/klog" ) -type driver struct { +type Driver struct { csiDriver *csicommon.CSIDriver endpoint string @@ -20,10 +20,10 @@ var ( DriverVersion = "latest" ) -func NewDriver(nodeID, endpoint string) *driver { +func NewDriver(nodeID, endpoint string) *Driver { klog.Infof("Starting new %s driver in version %s", DriverName, DriverVersion) - d := &driver{} + d := &Driver{} d.endpoint = endpoint @@ -34,13 +34,13 @@ func NewDriver(nodeID, endpoint string) *driver { return d } -func NewNodeServer(d *driver) *nodeServer { +func NewNodeServer(d *Driver) *nodeServer { return &nodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(d.csiDriver), } } -func (d *driver) Run() { +func (d *Driver) Run() { s := csicommon.NewNonBlockingGRPCServer() s.Start(d.endpoint, csicommon.NewDefaultIdentityServer(d.csiDriver), diff --git a/pkg/rclone/nodeserver.go b/pkg/rclone/nodeserver.go index 285752a..d01b5e7 100644 --- a/pkg/rclone/nodeserver.go +++ b/pkg/rclone/nodeserver.go @@ -1,11 +1,16 @@ package rclone import ( + "encoding/json" "fmt" "io/ioutil" + "net" + "net/http" "os" "os/exec" + "strconv" "strings" + "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,14 +27,34 @@ import ( csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) +type mountContext struct { + rcPort int +} + type nodeServer struct { + Driver *Driver *csicommon.DefaultNodeServer - mounter *mount.SafeFormatAndMount + mounter *mount.SafeFormatAndMount + mountContext map[string]*mountContext +} + +func (ns *nodeServer) getMountContext(targetPath string) *mountContext { + if mc, ok := ns.mountContext[targetPath]; ok { + return mc + } + return &mountContext{} +} + +func (ns *nodeServer) setMountContext(targetPath string, mc *mountContext) { + // create a new mount context + if ns.mountContext == nil { + ns.mountContext = make(map[string]*mountContext) + } + ns.mountContext[targetPath] = mc } -type mountPoint struct { - VolumeId string - MountPath string +func (ns *nodeServer) deleteMountContext(targetPath string) { + delete(ns.mountContext, targetPath) } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -75,7 +100,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } // Load default connection settings from secret - secret, e := getSecret("rclone-secret") + secret, _ := getSecret("rclone-secret") remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext(), secret) if e != nil { @@ -83,7 +108,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, e } - e = Mount(remote, remotePath, targetPath, configData, flags) + rcPort, e := Mount(remote, remotePath, targetPath, configData, flags) if e != nil { if os.IsPermission(e) { return nil, status.Error(codes.PermissionDenied, e.Error()) @@ -94,6 +119,11 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Error(codes.Internal, e.Error()) } + // Save the mount context + ns.setMountContext(targetPath, &mountContext{ + rcPort: rcPort, + }) + return &csi.NodePublishVolumeResponse{}, nil } @@ -144,15 +174,113 @@ func extractFlags(volumeContext map[string]string, secret *v1.Secret) (string, s return remote, remotePath, configData, flags, nil } -func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { +// https://rclone.org/rc/#core-stats +type rcCoreStatsResponse struct { + // an array of currently active file transfers + Transferring map[string]interface{} `json:"transferring"` +} + +// https://rclone.org/rc/#vfs-stats +type rcVfsStatsResponse struct { + DiskCache struct { + UploadsInProgress int64 `json:"uploadsInProgress"` + UploadsQueued int64 `json:"uploadsQueued"` + } `json:"diskCache"` +} + +// RcloneRPC is a helper function to call rclone rc server +func RcloneRPC(host string, method string, input string) (output string, err error) { + url := fmt.Sprintf("http://%s/%s", host, method) + + // Create a POST request to API + req, err := http.NewRequest("POST", url, strings.NewReader(input)) + if err != nil { + return "", fmt.Errorf("cannot create HTTP request: %v", err) + } + + // Set the content type to JSON + req.Header.Set("Content-Type", "application/json") + + // Create a new HTTP client + client := &http.Client{} + + // Send the request via the client + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("cannot send HTTP request: %v", err) + } + + // Close the response body on function exit + defer resp.Body.Close() + + // Read the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("cannot read HTTP response: %v", err) + } + + // Return the response body as a string + return string(body), nil +} - klog.Infof("NodeUnPublishVolume: called with args %+v", *req) +func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { targetPath := req.GetTargetPath() if len(targetPath) == 0 { return nil, status.Error(codes.InvalidArgument, "NodeUnpublishVolume Target Path must be provided") } + mountContext := ns.getMountContext(targetPath) + rcPort := mountContext.rcPort + + if rcPort != 0 { + // Connect to rclone rpc server and query the operation status + // If the rclone process is still running, wait for it to finish cache sync + // If the rclone process is not running, proceed to volume unmount + + // check the state of the rclone process until it finishes the cache sync + // Hard timeout is 1 hour + copyTimeout := time.Now().Add(1 * time.Hour) + for copyTimeout.After(time.Now()) { + + // Try to load https://localhost:5572/core/stats and parse the JSON response + out, err := RcloneRPC(fmt.Sprintf("localhost:%s", strconv.Itoa(rcPort)), "core/stats", "{}") + if err == nil { + var coreStats rcCoreStatsResponse + err = json.Unmarshal([]byte(out), &coreStats) + if err == nil { + if len(coreStats.Transferring) > 0 { + time.Sleep(5 * time.Second) + continue + } + } + + } + + // Try to load https://localhost:5572/vfs/stats and parse the JSON response + out, err = RcloneRPC(fmt.Sprintf("localhost:%s", strconv.Itoa(rcPort)), "vfs/stats", "{}") + if err == nil { + var vfsStats rcVfsStatsResponse + err = json.Unmarshal([]byte(out), &vfsStats) + if err == nil { + if vfsStats.DiskCache.UploadsInProgress > 0 || vfsStats.DiskCache.UploadsQueued > 0 { + time.Sleep(5 * time.Second) + continue + } + } + } + + // proceed to volume unmount + break + } + + // Remove VFS cache + os.RemoveAll("/tmp/rclone-vfs-cache/" + targetPath) + } + + // Remove mount context + ns.deleteMountContext(targetPath) + m := mount.New("") notMnt, err := m.IsLikelyNotMountPoint(targetPath) @@ -235,8 +363,21 @@ func flagToEnvName(flag string) string { return fmt.Sprintf("RCLONE_%s", flag) } +// Credit: https://gist.github.com/sevkin/96bdae9274465b2d09191384f86ef39d +func getFreePort() (port int, err error) { + var a *net.TCPAddr + if a, err = net.ResolveTCPAddr("tcp", "localhost:0"); err == nil { + var l *net.TCPListener + if l, err = net.ListenTCP("tcp", a); err == nil { + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil + } + } + return 0, err +} + // Mount routine. -func Mount(remote string, remotePath string, targetPath string, configData string, flags map[string]string) error { +func Mount(remote string, remotePath string, targetPath string, configData string, flags map[string]string) (rcPort int, err error) { mountCmd := "rclone" mountArgs := []string{} @@ -245,23 +386,33 @@ func Mount(remote string, remotePath string, targetPath string, configData strin defaultFlags["cache-chunk-clean-interval"] = "15m" defaultFlags["dir-cache-time"] = "5s" defaultFlags["vfs-cache-mode"] = "writes" + defaultFlags["cache-dir"] = "/tmp/rclone-vfs-cache/" + targetPath defaultFlags["allow-non-empty"] = "true" defaultFlags["allow-other"] = "true" remoteWithPath := fmt.Sprintf(":%s:%s", remote, remotePath) - if strings.Contains(configData, "[" + remote + "]") { + if strings.Contains(configData, "["+remote+"]") { remoteWithPath = fmt.Sprintf("%s:%s", remote, remotePath) klog.Infof("remote %s found in configData, remoteWithPath set to %s", remote, remoteWithPath) } + // Find a free port for rclone rc + rcPort, err = getFreePort() + if err != nil { + return 0, err + } + // rclone mount remote:path /path/to/mountpoint [flags] mountArgs = append( mountArgs, "mount", remoteWithPath, targetPath, + "--rc", + "--rc-addr="+fmt.Sprintf("localhost:%d", rcPort), "--daemon", + "--daemon-wait=0", ) // If a custom flag configData is defined, @@ -271,7 +422,7 @@ func Mount(remote string, remotePath string, targetPath string, configData strin configFile, err := ioutil.TempFile("", "rclone.conf") if err != nil { - return err + return 0, err } // Normally, a defer os.Remove(configFile.Name()) should be placed here. @@ -280,13 +431,16 @@ func Mount(remote string, remotePath string, targetPath string, configData strin // before it's reread by a forked process. if _, err := configFile.Write([]byte(configData)); err != nil { - return err + return 0, err } if err := configFile.Close(); err != nil { - return err + return 0, err } mountArgs = append(mountArgs, "--config", configFile.Name()) + } else { + // Disable "config not found" notice + mountArgs = append(mountArgs, "--config=''") } env := os.Environ() @@ -305,20 +459,21 @@ func Mount(remote string, remotePath string, targetPath string, configData strin } // create target, os.Mkdirall is noop if it exists - err := os.MkdirAll(targetPath, 0750) + err = os.MkdirAll(targetPath, 0750) if err != nil { - return err + return 0, err } klog.Infof("executing mount command cmd=%s, remote=%s, targetpath=%s", mountCmd, remoteWithPath, targetPath) + klog.Infof("mountArgs: %v", mountArgs) cmd := exec.Command(mountCmd, mountArgs...) cmd.Env = env out, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("mounting failed: %v cmd: '%s' remote: '%s' targetpath: %s output: %q", + return 0, fmt.Errorf("mounting failed: %v cmd: '%s' remote: '%s' targetpath: %s output: %q", err, mountCmd, remoteWithPath, targetPath, string(out)) } - return nil + return rcPort, nil }