Add rclone mounter (#15)

README.md

@@ -74,6 +74,7 @@ As S3 is not a real file system there are some limitations to consider here. Dep
 
 The driver can be configured to use one of these mounters to mount buckets:
 
+* [rclone](https://rclone.org/commands/rclone_mount)
 * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse)
 * [goofys](https://github.com/kahing/goofys)
 * [s3ql](https://github.com/s3ql/s3ql)
@@ -83,6 +84,10 @@ The mounter can be set as a parameter in the storage class. You can also create
 
 All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter:
 
+### rclone
+* Almost full POSIX compatibility (depends on caching mode)
+* Files can be viewed normally with any S3 client
+
 ### s3fs
 * Large subset of POSIX
 * Files can be viewed normally with any S3 client

cmd/s3driver/Dockerfile

@@ -44,7 +44,7 @@ LABEL description="csi-s3 production image"
 RUN apt-get update && \
     apt-get install -y \
       libfuse2 gcc sqlite3 libsqlite3-dev \
-      s3fs psmisc procps libcurl3 xfsprogs && \
+      s3fs psmisc procps libcurl3 xfsprogs wget unzip && \
     rm -rf /var/lib/apt/lists/*
 
 ARG S3QL_VERSION=2.29
@@ -56,5 +56,13 @@ RUN pip install ${S3QL_URL} && rm -rf /root/.cache
 
 COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer
 
+# install rclone
+ARG RCLONE_VERSION=v1.46
+RUN cd /tmp \
+    && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
+    && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
+    && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
+    && rm -r /tmp/rclone*
+
 COPY ./_output/s3driver /s3driver
 ENTRYPOINT ["/s3driver"]

go.mod

@@ -1,10 +1,9 @@
-module github.com/ctrox/csi-s3/cmd/s3driver
+module github.com/ctrox/csi-s3
 
 require (
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/aws/aws-sdk-go v1.14.27 // indirect
 	github.com/container-storage-interface/spec v1.0.0
-	github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096
 	github.com/go-ini/ini v1.38.1 // indirect
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b

go.sum

@@ -4,8 +4,7 @@ github.com/aws/aws-sdk-go v1.14.27 h1:fRVME5X3sxZnctdCcabNTWZq7ZGrpVgUAYk4OA5EG0
 github.com/aws/aws-sdk-go v1.14.27/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E=
 github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
-github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096 h1:68EqNg4/ll/Np4mwA4anaM5IlriPqjBaWp3rHJ2wwoM=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
-github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096/go.mod h1:1CC5YEd7YfNfwmVtjA1okhZTrgYcJCScAllqnYF6fVc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY=
 github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
@@ -15,6 +14,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc=
 github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb h1:TRAjtOoio6kvnrIMLeXesGT9IydfO+zQoioKWrv40nI=
 github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU=
@@ -22,7 +22,9 @@ github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbd
 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/kahing/go-xattr v1.1.1 h1:7Ft/P9Gc6iqRVzBRLVw/yLL/dbtzL6FsZzGQj3T9ZY8=
 github.com/kahing/go-xattr v1.1.1/go.mod h1:DXZs3JwPmH2DnyFxWjLZWb65lq8pOPtsf9LD+2Gbbpw=
 github.com/kahing/goofys v0.19.0 h1:jcuffrnpvZq+LjXtRODo0pvNOglw32ClzBZ1XLShFnk=
 github.com/kahing/goofys v0.19.0/go.mod h1:erC9E45nY5m8v6FE+tYIGRVjIC2N8viMlJrgrsXB2Q4=
@@ -30,6 +32,7 @@ github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdT
 github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0=
 github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e h1:BkkRJIF329ps8digiMWthYzDPl9KB8PwkDwvVWDwM4A=
 github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrMenWrjM6nZh0Q=
 github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
@@ -41,15 +44,20 @@ github.com/onsi/ginkgo v1.5.0 h1:uZr+v/TFDdYkdA+j02sPO1kA5owrfjBGCJAogfIyThE=
 github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.0 h1:p/ZBjQI9G/VwoPrslo/sqS6R5vHU9Od60+axIiP6WuQ=
 github.com/onsi/gomega v1.4.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f h1:lv02BiKkf3A85oirJHx0feXbKV4xrq5Nf7QbrNyILoo=
 github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
+github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
 github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
@@ -57,6 +65,7 @@ golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 h1:h7zdf0RiEvWbYBKIx4b+q4
 golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20180712202826-d0887baf81f4 h1:KDF3PK6A+dkI7c4O8QbMtJqcXE3LdNJFGZECIlifQOg=
 golang.org/x/net v0.0.0-20180712202826-d0887baf81f4/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180715085529-ac767d655b30 h1:4bYUqrXBoiI7UFQeibUwFhvcHfaEeL75O3lOcZa964o=
 golang.org/x/sys v0.0.0-20180715085529-ac767d655b30/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -66,8 +75,11 @@ google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b h1:mXqBiicV0B+k8wz
 google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.13.0 h1:bHIbVsCwmvbArgCJmLdgOdHFXlKqTOVjbibbS19cXHc=
 google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/ini.v1 v1.38.1 h1:8E3nEICVJ6kxl6aTXYp77xYyObhw7YG9/avdj0r3vME=
 gopkg.in/ini.v1 v1.38.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

pkg/s3/mounter.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"fmt"
 	"os/exec"
+	"time"
 
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/util/mount"
@@ -22,6 +23,7 @@ const (
 	goofysMounterType   = "goofys"
 	s3qlMounterType     = "s3ql"
 	s3backerMounterType = "s3backer"
+	rcloneMounterType   = "rclone"
 	mounterTypeKey      = "mounter"
 )
 
@@ -45,6 +47,9 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) {
 	case s3backerMounterType:
 		return newS3backerMounter(bucket, cfg)
 
+	case rcloneMounterType:
+		return newRcloneMounter(bucket, cfg)
+
 	default:
 		// default to s3backer
 		return newS3backerMounter(bucket, cfg)
@@ -59,7 +64,7 @@ func fuseMount(path string, command string, args []string) error {
 		return fmt.Errorf("Error fuseMount command: %s\nargs: %s\noutput: %s", command, args, out)
 	}
 
-	return nil
+	return waitForMount(path, 10*time.Second)
 }
 
 func fuseUnmount(path string, command string) error {

pkg/s3/mounter_rclone.go

@@ -0,0 +1,60 @@
+package s3
+
+import (
+	"fmt"
+	"os"
+)
+
+// Implements Mounter
+type rcloneMounter struct {
+	bucket          *bucket
+	url             string
+	region          string
+	accessKeyID     string
+	secretAccessKey string
+}
+
+const (
+	rcloneCmd = "rclone"
+)
+
+func newRcloneMounter(b *bucket, cfg *Config) (Mounter, error) {
+	return &rcloneMounter{
+		bucket:          b,
+		url:             cfg.Endpoint,
+		region:          cfg.Region,
+		accessKeyID:     cfg.AccessKeyID,
+		secretAccessKey: cfg.SecretAccessKey,
+	}, nil
+}
+
+func (rclone *rcloneMounter) Stage(stageTarget string) error {
+	return nil
+}
+
+func (rclone *rcloneMounter) Unstage(stageTarget string) error {
+	return nil
+}
+
+func (rclone *rcloneMounter) Mount(source string, target string) error {
+	args := []string{
+		"mount",
+		fmt.Sprintf(":s3:%s/%s", rclone.bucket.Name, rclone.bucket.FSPath),
+		fmt.Sprintf("%s", target),
+		"--daemon",
+		"--s3-provider=AWS",
+		"--s3-env-auth=true",
+		fmt.Sprintf("--s3-region=%s", rclone.region),
+		fmt.Sprintf("--s3-endpoint=%s", rclone.url),
+		"--allow-other",
+		// TODO: make this configurable
+		"--vfs-cache-mode=writes",
+	}
+	os.Setenv("AWS_ACCESS_KEY_ID", rclone.accessKeyID)
+	os.Setenv("AWS_SECRET_ACCESS_KEY", rclone.secretAccessKey)
+	return fuseMount(target, rcloneCmd, args)
+}
+
+func (rclone *rcloneMounter) Unmount(target string) error {
+	return fuseUnmount(target, rcloneCmd)
+}

pkg/s3/s3-driver_suite_test.go

@@ -139,4 +139,32 @@ var _ = Describe("S3Driver", func() {
 		})
 	})
 
+	Context("rclone", func() {
+		socket := "/tmp/csi-rclone.sock"
+		csiEndpoint := "unix://" + socket
+
+		cfg := &s3.Config{
+			AccessKeyID:     "FJDSJ",
+			SecretAccessKey: "DSG643HGDS",
+			Endpoint:        "http://127.0.0.1:9000",
+			Mounter:         "rclone",
+		}
+		if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		driver, err := s3.NewS3("test-node", csiEndpoint, cfg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		go driver.Run()
+
+		Describe("CSI sanity", func() {
+			sanityCfg := &sanity.Config{
+				TargetPath:  mntDir,
+				StagingPath: stagingDir,
+				Address:     csiEndpoint,
+			}
+			sanity.GinkgoTest(sanityCfg)
+		})
+	})
 })

pkg/s3/util.go

@@ -1,6 +1,7 @@
 package s3
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -10,6 +11,7 @@ import (
 	"time"
 
 	"github.com/mitchellh/go-ps"
+	"k8s.io/kubernetes/pkg/util/mount"
 
 	"github.com/golang/glog"
 )
@@ -39,6 +41,25 @@ func waitForProcess(p *os.Process, backoff int) error {
 	return waitForProcess(p, backoff+1)
 }
 
+func waitForMount(path string, timeout time.Duration) error {
+	var elapsed time.Duration
+	var interval = 10 * time.Millisecond
+	for {
+		notMount, err := mount.New("").IsNotMountPoint(path)
+		if err != nil {
+			return err
+		}
+		if !notMount {
+			return nil
+		}
+		time.Sleep(interval)
+		elapsed = elapsed + interval
+		if elapsed >= timeout {
+			return errors.New("Timeout waiting for mount")
+		}
+	}
+}
+
 func findFuseMountProcess(path string, name string) (*os.Process, error) {
 	processes, err := ps.Processes()
 	if err != nil {

test/Dockerfile

@@ -1,4 +1,4 @@
-FROM ctrox/csi-s3:1.0.1-alpha
+FROM ctrox/csi-s3:dev
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
 LABEL description="csi-s3 testing image"