From ea4022e9c69c1366e07ad894947a622bd94528a5 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Thu, 7 Mar 2019 20:27:02 +0100 Subject: [PATCH] Add rclone mounter (#15) --- README.md | 5 +++ cmd/s3driver/Dockerfile | 10 +++++- go.mod | 3 +- go.sum | 16 +++++++-- pkg/s3/mounter.go | 7 +++- pkg/s3/mounter_rclone.go | 60 ++++++++++++++++++++++++++++++++++ pkg/s3/s3-driver_suite_test.go | 28 ++++++++++++++++ pkg/s3/util.go | 21 ++++++++++++ test/Dockerfile | 2 +- 9 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 pkg/s3/mounter_rclone.go diff --git a/README.md b/README.md index 416159f..1783ac9 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ As S3 is not a real file system there are some limitations to consider here. Dep The driver can be configured to use one of these mounters to mount buckets: +* [rclone](https://rclone.org/commands/rclone_mount) * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) * [goofys](https://github.com/kahing/goofys) * [s3ql](https://github.com/s3ql/s3ql) @@ -83,6 +84,10 @@ The mounter can be set as a parameter in the storage class. You can also create All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter: +### rclone +* Almost full POSIX compatibility (depends on caching mode) +* Files can be viewed normally with any S3 client + ### s3fs * Large subset of POSIX * Files can be viewed normally with any S3 client diff --git a/cmd/s3driver/Dockerfile b/cmd/s3driver/Dockerfile index 918d998..f6c94f9 100644 --- a/cmd/s3driver/Dockerfile +++ b/cmd/s3driver/Dockerfile @@ -44,7 +44,7 @@ LABEL description="csi-s3 production image" RUN apt-get update && \ apt-get install -y \ libfuse2 gcc sqlite3 libsqlite3-dev \ - s3fs psmisc procps libcurl3 xfsprogs && \ + s3fs psmisc procps libcurl3 xfsprogs wget unzip && \ rm -rf /var/lib/apt/lists/* ARG S3QL_VERSION=2.29 @@ -56,5 +56,13 @@ RUN pip install ${S3QL_URL} && rm -rf /root/.cache COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer +# install rclone +ARG RCLONE_VERSION=v1.46 +RUN cd /tmp \ + && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ + && rm -r /tmp/rclone* + COPY ./_output/s3driver /s3driver ENTRYPOINT ["/s3driver"] diff --git a/go.mod b/go.mod index d966032..0283831 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ -module github.com/ctrox/csi-s3/cmd/s3driver +module github.com/ctrox/csi-s3 require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/aws/aws-sdk-go v1.14.27 // indirect github.com/container-storage-interface/spec v1.0.0 - github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096 github.com/go-ini/ini v1.38.1 // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b diff --git a/go.sum b/go.sum index 9f244b9..38372b6 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,7 @@ github.com/aws/aws-sdk-go v1.14.27 h1:fRVME5X3sxZnctdCcabNTWZq7ZGrpVgUAYk4OA5EG0 github.com/aws/aws-sdk-go v1.14.27/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/container-storage-interface/spec v1.0.0 h1:3DyXuJgf9MU6kyULESegQUmozsSxhpyrrv9u5bfwA3E= github.com/container-storage-interface/spec v1.0.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096 h1:68EqNg4/ll/Np4mwA4anaM5IlriPqjBaWp3rHJ2wwoM= -github.com/ctrox/csi-s3 v0.0.0-20190228183403-83723f4be096/go.mod h1:1CC5YEd7YfNfwmVtjA1okhZTrgYcJCScAllqnYF6fVc= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ini/ini v1.38.1 h1:hbtfM8emWUVo9GnXSloXYyFbXxZ+tG6sbepSStoe1FY= github.com/go-ini/ini v1.38.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -15,6 +14,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb h1:TRAjtOoio6kvnrIMLeXesGT9IydfO+zQoioKWrv40nI= github.com/jacobsa/fuse v0.0.0-20180417054321-cd3959611bcb/go.mod h1:9Aml1MG17JVeXrN4D2mtJvYHtHklJH5bESjCKNzVjFU= @@ -22,7 +22,9 @@ github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbd github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kahing/go-xattr v1.1.1 h1:7Ft/P9Gc6iqRVzBRLVw/yLL/dbtzL6FsZzGQj3T9ZY8= github.com/kahing/go-xattr v1.1.1/go.mod h1:DXZs3JwPmH2DnyFxWjLZWb65lq8pOPtsf9LD+2Gbbpw= github.com/kahing/goofys v0.19.0 h1:jcuffrnpvZq+LjXtRODo0pvNOglw32ClzBZ1XLShFnk= github.com/kahing/goofys v0.19.0/go.mod h1:erC9E45nY5m8v6FE+tYIGRVjIC2N8viMlJrgrsXB2Q4= @@ -30,6 +32,7 @@ github.com/kubernetes-csi/csi-test v1.1.0 h1:a7CfGqhGDs0h7AZt1f6LTIUzBazcRf6eBdT github.com/kubernetes-csi/csi-test v1.1.0/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e h1:BkkRJIF329ps8digiMWthYzDPl9KB8PwkDwvVWDwM4A= github.com/kubernetes-csi/drivers v0.0.0-20181207022357-c1e71bdcce6e/go.mod h1:V6rHbbSLCZGaQoIZ8MkyDtoXtcKXZM0F7N3bkloDCOY= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrMenWrjM6nZh0Q= github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= @@ -41,15 +44,20 @@ github.com/onsi/ginkgo v1.5.0 h1:uZr+v/TFDdYkdA+j02sPO1kA5owrfjBGCJAogfIyThE= github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.0 h1:p/ZBjQI9G/VwoPrslo/sqS6R5vHU9Od60+axIiP6WuQ= github.com/onsi/gomega v1.4.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f h1:lv02BiKkf3A85oirJHx0feXbKV4xrq5Nf7QbrNyILoo= github.com/shirou/gopsutil v0.0.0-20180625081143-4a180b209f5f/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -57,6 +65,7 @@ golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8 h1:h7zdf0RiEvWbYBKIx4b+q4 golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/net v0.0.0-20180712202826-d0887baf81f4 h1:KDF3PK6A+dkI7c4O8QbMtJqcXE3LdNJFGZECIlifQOg= golang.org/x/net v0.0.0-20180712202826-d0887baf81f4/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180715085529-ac767d655b30 h1:4bYUqrXBoiI7UFQeibUwFhvcHfaEeL75O3lOcZa964o= golang.org/x/sys v0.0.0-20180715085529-ac767d655b30/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -66,8 +75,11 @@ google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b h1:mXqBiicV0B+k8wz google.golang.org/genproto v0.0.0-20180716172848-2731d4fa720b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.13.0 h1:bHIbVsCwmvbArgCJmLdgOdHFXlKqTOVjbibbS19cXHc= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/ini.v1 v1.38.1 h1:8E3nEICVJ6kxl6aTXYp77xYyObhw7YG9/avdj0r3vME= gopkg.in/ini.v1 v1.38.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index 0135ef6..6bddc39 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -3,6 +3,7 @@ package s3 import ( "fmt" "os/exec" + "time" "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/mount" @@ -22,6 +23,7 @@ const ( goofysMounterType = "goofys" s3qlMounterType = "s3ql" s3backerMounterType = "s3backer" + rcloneMounterType = "rclone" mounterTypeKey = "mounter" ) @@ -45,6 +47,9 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) { case s3backerMounterType: return newS3backerMounter(bucket, cfg) + case rcloneMounterType: + return newRcloneMounter(bucket, cfg) + default: // default to s3backer return newS3backerMounter(bucket, cfg) @@ -59,7 +64,7 @@ func fuseMount(path string, command string, args []string) error { return fmt.Errorf("Error fuseMount command: %s\nargs: %s\noutput: %s", command, args, out) } - return nil + return waitForMount(path, 10*time.Second) } func fuseUnmount(path string, command string) error { diff --git a/pkg/s3/mounter_rclone.go b/pkg/s3/mounter_rclone.go new file mode 100644 index 0000000..60d1203 --- /dev/null +++ b/pkg/s3/mounter_rclone.go @@ -0,0 +1,60 @@ +package s3 + +import ( + "fmt" + "os" +) + +// Implements Mounter +type rcloneMounter struct { + bucket *bucket + url string + region string + accessKeyID string + secretAccessKey string +} + +const ( + rcloneCmd = "rclone" +) + +func newRcloneMounter(b *bucket, cfg *Config) (Mounter, error) { + return &rcloneMounter{ + bucket: b, + url: cfg.Endpoint, + region: cfg.Region, + accessKeyID: cfg.AccessKeyID, + secretAccessKey: cfg.SecretAccessKey, + }, nil +} + +func (rclone *rcloneMounter) Stage(stageTarget string) error { + return nil +} + +func (rclone *rcloneMounter) Unstage(stageTarget string) error { + return nil +} + +func (rclone *rcloneMounter) Mount(source string, target string) error { + args := []string{ + "mount", + fmt.Sprintf(":s3:%s/%s", rclone.bucket.Name, rclone.bucket.FSPath), + fmt.Sprintf("%s", target), + "--daemon", + "--s3-provider=AWS", + "--s3-env-auth=true", + fmt.Sprintf("--s3-region=%s", rclone.region), + fmt.Sprintf("--s3-endpoint=%s", rclone.url), + "--allow-other", + // TODO: make this configurable + "--vfs-cache-mode=writes", + } + os.Setenv("AWS_ACCESS_KEY_ID", rclone.accessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", rclone.secretAccessKey) + return fuseMount(target, rcloneCmd, args) +} + +func (rclone *rcloneMounter) Unmount(target string) error { + return fuseUnmount(target, rcloneCmd) +} diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go index eb77f5c..8fbe0f7 100644 --- a/pkg/s3/s3-driver_suite_test.go +++ b/pkg/s3/s3-driver_suite_test.go @@ -139,4 +139,32 @@ var _ = Describe("S3Driver", func() { }) }) + Context("rclone", func() { + socket := "/tmp/csi-rclone.sock" + csiEndpoint := "unix://" + socket + + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "rclone", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + StagingPath: stagingDir, + Address: csiEndpoint, + } + sanity.GinkgoTest(sanityCfg) + }) + }) }) diff --git a/pkg/s3/util.go b/pkg/s3/util.go index d85d86c..e0bc29f 100644 --- a/pkg/s3/util.go +++ b/pkg/s3/util.go @@ -1,6 +1,7 @@ package s3 import ( + "errors" "fmt" "io/ioutil" "os" @@ -10,6 +11,7 @@ import ( "time" "github.com/mitchellh/go-ps" + "k8s.io/kubernetes/pkg/util/mount" "github.com/golang/glog" ) @@ -39,6 +41,25 @@ func waitForProcess(p *os.Process, backoff int) error { return waitForProcess(p, backoff+1) } +func waitForMount(path string, timeout time.Duration) error { + var elapsed time.Duration + var interval = 10 * time.Millisecond + for { + notMount, err := mount.New("").IsNotMountPoint(path) + if err != nil { + return err + } + if !notMount { + return nil + } + time.Sleep(interval) + elapsed = elapsed + interval + if elapsed >= timeout { + return errors.New("Timeout waiting for mount") + } + } +} + func findFuseMountProcess(path string, name string) (*os.Process, error) { processes, err := ps.Processes() if err != nil { diff --git a/test/Dockerfile b/test/Dockerfile index f4ea0ca..26ea2a0 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,4 +1,4 @@ -FROM ctrox/csi-s3:1.0.1-alpha +FROM ctrox/csi-s3:dev LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 testing image"