Merge branch 's3ql_mounter'

Gopkg.lock

@@ -125,13 +125,13 @@
   version = "v0.19.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/kubernetes-csi/csi-test"
   packages = [
     "pkg/sanity",
     "utils"
   ]
-  revision = "6fed82d24d3a04c1814440d4178e7bf8ae9e67e6"
+  revision = "718c9544f5e16cba31881333e0f9dff371663dea"
+  version = "v0.3.0-1"
 
 [[projects]]
   branch = "master"
@@ -334,6 +334,12 @@
   revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
   version = "v1.13.0"
 
+[[projects]]
+  name = "gopkg.in/ini.v1"
+  packages = ["."]
+  revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
+  version = "v1.38.1"
+
 [[projects]]
   name = "gopkg.in/yaml.v2"
   packages = ["."]
@@ -366,6 +372,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4"
+  inputs-digest = "659f47734b56af7fba146039231841e82cc4c3c95dff2814ec3688e967790a50"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -11,8 +11,8 @@
   version = "0.19.0"
 
 [[constraint]]
-  branch = "master"
   name = "github.com/kubernetes-csi/csi-test"
+  version = "v0.3.0-1"
 
 [[constraint]]
   branch = "master"
@@ -37,3 +37,7 @@
 [prune]
   go-tests = true
   unused-packages = true
+
+[[constraint]]
+  name = "gopkg.in/ini.v1"
+  version = "1.38.1"

Makefile

@@ -16,7 +16,7 @@
 PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver
 REGISTRY_NAME=ctrox
 IMAGE_NAME=csi-s3-driver
-IMAGE_VERSION=0.1.0
+IMAGE_VERSION=0.2.0
 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
 TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test
 
@@ -27,7 +27,7 @@ test:
 	docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile .
 	docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG)
 container: build
-	docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile .
+	docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql .
 push: container
 	docker push $(IMAGE_TAG)
 clean:

README.md

@@ -1,6 +1,9 @@
 # CSI for S3
 This is a Container Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) for S3 (or S3 compatible) storage. This can dynamically allocate buckets and mount them via a fuse mount into any container.
 
+# Status
+This is still very experimental and should not be used in any production environment. Unexpected data loss could occur depending on what mounter and S3 storage backend is being used.
+
 # Kubernetes installation
 ## Requirements
 * Kubernetes 1.10+
@@ -19,6 +22,12 @@ stringData:
   secretAccessKey: <YOUR_SECRET_ACCES_KEY>
   endpoint: <S3_ENDPOINT_URL
   region: <S3_REGION>
+  # specify which mounter to use
+  # can be set to s3fs, goofys or s3ql
+  mounter: <MOUNTER>
+  # Currently only for s3ql
+  # If not using s3ql, set it to ""
+  encryptionKey: <FS_ENCRYPTION_KEY>
 ```
 
 ## 2. Deploy the driver
@@ -62,20 +71,32 @@ If something does not work as expected, check the troubleshooting section below.
 
 # Additional configuration
 ## Mounter
-By default the driver will use [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) to mount buckets. Alternatively you can configure the storage class to use [goofys](https://github.com/kahing/goofys) for mounting S3 buckets. Note that goofys has some drawbacks in regards to POSIX compliance but in return offers better Performance than s3fs.
+As seen in the deployment example above, the driver can be configured to use one of these mounters to mount buckets:
 
-To configure a storage class to use goofys, just set the `mounter` parameter to `goofys`
+* [s3fs](https://github.com/s3fs-fuse/s3fs-fuse)
-```yaml
+* [goofys](https://github.com/kahing/goofys)
-kind: StorageClass
+* [s3ql](https://github.com/s3ql/s3ql)
-apiVersion: storage.k8s.io/v1
+
-metadata:
+All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter:
-  name: csi-s3
+
-provisioner: ch.ctrox.csi.s3-driver
+### s3fs
-parameters:
+* Large subset of POSIX
-  mounter: goofys
+* Files can be viewed normally with any S3 client
-  csiProvisionerSecretName: csi-s3-secret
+* Does not support appends or random writes
-  csiProvisionerSecretNamespace: kube-system
+
-```
+### goofys
+* Weak POSIX compatibility
+* Performance first
+* Files can be viewed normally with any S3 client
+* Does not support appends or random writes
+
+### s3ql
+* (Almost) full POSIX compatibility
+* Uses its own object format
+* Files are not readable with other S3 clients
+* Support appends
+* Supports compression before upload
+* Supports encryption before upload
 
 # Limitations
 As S3 is not a real file system there are some limitations to consider here. Depending on what mounter you are using, you will have different levels of POSIX compability. Also depending on what S3 storage backend you are using there are not always consistency guarantees. The detailed limitations can be found on the documentation of [s3fs](https://github.com/s3fs-fuse/s3fs-fuse#limitations) and [goofys](https://github.com/kahing/goofys#current-status).

cmd/s3driver/Dockerfile.s3ql

@@ -0,0 +1,23 @@
+FROM debian:stretch
+LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
+LABEL description="s3 fuse csi plugin"
+ARG S3QL_VERSION=release-2.28
+
+RUN apt-get update && \
+    apt-get install -y \
+      s3fs wget psmisc procps python3 python3-setuptools \
+      python3-dev python3-pip python3-llfuse pkg-config \
+      sqlite3 libsqlite3-dev python3-apsw cython && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install defusedxml dugong requests pycrypto
+
+WORKDIR /usr/src
+RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
+RUN tar -xzf ${S3QL_VERSION}.tar.gz
+WORKDIR /usr/src/s3ql-${S3QL_VERSION}
+RUN python3 setup.py build_cython build_ext --inplace
+RUN python3 setup.py install
+
+COPY ./_output/s3driver /s3driver
+ENTRYPOINT ["/s3driver"]

cmd/s3driver/main.go

@@ -35,6 +35,8 @@ var (
 	secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use")
 	s3endpoint      = flag.String("s3-endpoint", "", "S3 Endpoint URL to use")
 	region          = flag.String("region", "", "S3 Region to use")
+	mounter         = flag.String("mounter", "s3fs", "Specify which Mounter to use")
+	encryptionKey   = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)")
 )
 
 func main() {
@@ -45,6 +47,8 @@ func main() {
 		SecretAccessKey: *secretAccessKey,
 		Endpoint:        *s3endpoint,
 		Region:          *region,
+		Mounter:         *mounter,
+		EncryptionKey:   *encryptionKey,
 	}
 
 	driver, err := s3.NewS3(*nodeID, *endpoint, cfg)

deploy/kubernetes/csi-s3-driver.yaml

@@ -75,7 +75,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: ctrox/csi-s3-driver:0.1.0
+          image: ctrox/csi-s3-driver:0.2.0
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(NODE_ID)"
@@ -83,6 +83,8 @@ spec:
             - "--secret-access-key=$(SECRET_ACCESS_KEY)"
             - "--s3-endpoint=$(S3_ENDPOINT)"
             - "--region=$(REGION)"
+            - "--mounter=$(MOUNTER)"
+            - "--encryption-key=$(ENCRYPTION_KEY)"
             - "--v=4"
           env:
             - name: CSI_ENDPOINT
@@ -111,6 +113,16 @@ spec:
                 secretKeyRef:
                   name: csi-s3-secret
                   key: region
+            - name: MOUNTER
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: mounter
+            - name: ENCRYPTION_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: encryptionKey
           imagePullPolicy: "Always"
           volumeMounts:
             - name: plugin-dir

deploy/kubernetes/provisioner.yaml

@@ -79,7 +79,7 @@ spec:
             - name: socket-dir
               mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver
         - name: s3-csi-driver
-          image: ctrox/csi-s3-driver:0.1.0
+          image: ctrox/csi-s3-driver:0.2.0
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(NODE_ID)"
@@ -87,6 +87,8 @@ spec:
             - "--secret-access-key=$(SECRET_ACCESS_KEY)"
             - "--s3-endpoint=$(S3_ENDPOINT)"
             - "--region=$(REGION)"
+            - "--mounter=$(MOUNTER)"
+            - "--encryption-key=$(ENCRYPTION_KEY)"
             - "--v=4"
           env:
             - name: CSI_ENDPOINT
@@ -115,6 +117,16 @@ spec:
                 secretKeyRef:
                   name: csi-s3-secret
                   key: region
+            - name: MOUNTER
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: mounter
+            - name: ENCRYPTION_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: encryptionKey
           imagePullPolicy: "Always"
           volumeMounts:
             - name: socket-dir

deploy/kubernetes/pvc.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: default
 spec:
   accessModes:
-  - ReadWriteMany
+  - ReadWriteOnce
   resources:
     requests:
       storage: 5Gi

deploy/kubernetes/secret.yaml

@@ -6,5 +6,11 @@ stringData:
   accessKeyID: <YOUR_ACCESS_KEY_ID>
   secretAccessKey: <YOUR_SECRET_ACCES_KEY>
   endpoint: <S3_ENDPOINT_URL>
-  # If not on S3, just set it to ""
+  # If not on S3, set it to ""
   region: <S3_REGION>
+  # specify which mounter to use
+  # can be set to s3fs, goofys or s3ql
+  mounter: <MOUNTER>
+  # Currently only for s3ql
+  # If not using s3ql, set it to ""
+  encryptionKey: <FS_ENCRYPTION_KEY>

deploy/kubernetes/storageclass.yaml

@@ -4,8 +4,3 @@ apiVersion: storage.k8s.io/v1
 metadata:
   name: csi-s3
 provisioner: ch.ctrox.csi.s3-driver
-parameters:
-  # specify which mounter to use
-  # can be set to s3fs or goofys
-  # s3fs is the default
-  # mounter: s3fs

pkg/s3/config.go

@@ -6,4 +6,6 @@ type Config struct {
 	SecretAccessKey string
 	Region          string
 	Endpoint        string
+	Mounter         string
+	EncryptionKey   string
 }

pkg/s3/controllerserver.go

@@ -55,12 +55,20 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}
 	if !exists {
-		if err := cs.s3.client.createBucket(volumeID); err != nil {
+		if err = cs.s3.client.createBucket(volumeID); err != nil {
 			glog.V(3).Infof("failed to create volume: %v", err)
 			return nil, err
 		}
 	}
 
+	mounter, err := newMounter(volumeID, cs.s3.cfg)
+	if err != nil {
+		return nil, err
+	}
+	if err := mounter.Format(); err != nil {
+		return nil, err
+	}
+
 	glog.V(4).Infof("create volume %s", volumeID)
 	s3Vol := s3Volume{}
 	s3Vol.VolName = req.GetName()
@@ -68,7 +76,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			Id:            volumeID,
-			CapacityBytes: 0,
+			CapacityBytes: 1,
 			Attributes:    req.GetParameters(),
 		},
 	}, nil
@@ -123,6 +131,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
 		return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId()))
 	}
 
-	// We currently support all capabilities
+	for _, cap := range req.VolumeCapabilities {
-	return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil
+		if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+			return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
+		}
+	}
+	return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil
 }

pkg/s3/goofys.go

@@ -1,37 +0,0 @@
-package s3
-
-import (
-	"fmt"
-	"os"
-
-	"context"
-
-	goofys "github.com/kahing/goofys/api"
-)
-
-const defaultRegion = "us-east-1"
-
-func goofysMount(bucket string, cfg *Config, targetPath string) error {
-	goofysCfg := &goofys.Config{
-		MountPoint: targetPath,
-		Endpoint:   cfg.Endpoint,
-		Region:     cfg.Region,
-		DirMode:    0755,
-		FileMode:   0644,
-		MountOptions: map[string]string{
-			"allow_other": "",
-		},
-	}
-	if cfg.Endpoint != "" {
-		cfg.Region = defaultRegion
-	}
-	os.Setenv("AWS_ACCESS_KEY_ID", cfg.AccessKeyID)
-	os.Setenv("AWS_SECRET_ACCESS_KEY", cfg.SecretAccessKey)
-
-	_, _, err := goofys.Mount(context.Background(), bucket, goofysCfg)
-
-	if err != nil {
-		return fmt.Errorf("Error mounting via goofys: %s", err)
-	}
-	return nil
-}

pkg/s3/mounter.go

@@ -0,0 +1,33 @@
+package s3
+
+import "fmt"
+
+// Mounter interface which can be implemented
+// by the different mounter types
+type Mounter interface {
+	Format() error
+	Mount(targetPath string) error
+	Unmount(targetPath string) error
+}
+
+const (
+	s3fsMounterType   = "s3fs"
+	goofysMounterType = "goofys"
+	s3qlMounterType   = "s3ql"
+)
+
+// newMounter returns a new mounter depending on the mounterType parameter
+func newMounter(bucket string, cfg *Config) (Mounter, error) {
+	switch cfg.Mounter {
+	case s3fsMounterType:
+		return newS3fsMounter(bucket, cfg)
+
+	case goofysMounterType:
+		return newGoofysMounter(bucket, cfg)
+
+	case s3qlMounterType:
+		return newS3qlMounter(bucket, cfg)
+
+	}
+	return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", bucket, cfg.Mounter)
+}

pkg/s3/mounter_goofys.go

@@ -0,0 +1,68 @@
+package s3
+
+import (
+	"fmt"
+	"os"
+
+	"context"
+
+	goofysApi "github.com/kahing/goofys/api"
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+const defaultRegion = "us-east-1"
+
+// Implements Mounter
+type goofysMounter struct {
+	bucket          string
+	endpoint        string
+	region          string
+	accessKeyID     string
+	secretAccessKey string
+}
+
+func newGoofysMounter(bucket string, cfg *Config) (Mounter, error) {
+	region := cfg.Region
+	// if endpoint is set we need a default region
+	if region == "" && cfg.Endpoint != "" {
+		region = defaultRegion
+	}
+	return &goofysMounter{
+		bucket:          bucket,
+		endpoint:        cfg.Endpoint,
+		region:          region,
+		accessKeyID:     cfg.AccessKeyID,
+		secretAccessKey: cfg.SecretAccessKey,
+	}, nil
+}
+
+func (goofys *goofysMounter) Format() error {
+	return nil
+}
+
+func (goofys *goofysMounter) Mount(targetPath string) error {
+	goofysCfg := &goofysApi.Config{
+		MountPoint: targetPath,
+		Endpoint:   goofys.endpoint,
+		Region:     goofys.region,
+		DirMode:    0755,
+		FileMode:   0644,
+		MountOptions: map[string]string{
+			"allow_other": "",
+		},
+	}
+
+	os.Setenv("AWS_ACCESS_KEY_ID", goofys.accessKeyID)
+	os.Setenv("AWS_SECRET_ACCESS_KEY", goofys.secretAccessKey)
+
+	_, _, err := goofysApi.Mount(context.Background(), goofys.bucket, goofysCfg)
+
+	if err != nil {
+		return fmt.Errorf("Error mounting via goofys: %s", err)
+	}
+	return nil
+}
+
+func (goofys *goofysMounter) Unmount(targetPath string) error {
+	return mount.New("").Unmount(targetPath)
+}

pkg/s3/mounter_s3fs.go

@@ -0,0 +1,70 @@
+package s3
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+// Implements Mounter
+type s3fsMounter struct {
+	bucket        string
+	url           string
+	region        string
+	pwFileContent string
+}
+
+func newS3fsMounter(bucket string, cfg *Config) (Mounter, error) {
+	return &s3fsMounter{
+		bucket:        bucket,
+		url:           cfg.Endpoint,
+		region:        cfg.Region,
+		pwFileContent: cfg.AccessKeyID + ":" + cfg.SecretAccessKey,
+	}, nil
+}
+
+func (s3fs *s3fsMounter) Format() error {
+	return nil
+}
+
+func (s3fs *s3fsMounter) Mount(targetPath string) error {
+	if err := writes3fsPass(s3fs.pwFileContent); err != nil {
+		return err
+	}
+	args := []string{
+		fmt.Sprintf("%s", s3fs.bucket),
+		fmt.Sprintf("%s", targetPath),
+		"-o", "sigv2",
+		"-o", "use_path_request_style",
+		"-o", fmt.Sprintf("url=%s", s3fs.url),
+		"-o", fmt.Sprintf("endpoint=%s", s3fs.region),
+		"-o", "allow_other",
+		"-o", "mp_umask=000",
+	}
+	cmd := exec.Command("s3fs", args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Error mounting using s3fs, output: %s", out)
+	}
+	return nil
+}
+
+func (s3fs *s3fsMounter) Unmount(targetPath string) error {
+	return mount.New("").Unmount(targetPath)
+}
+
+func writes3fsPass(pwFileContent string) error {
+	pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME"))
+	pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+	_, err = pwFile.WriteString(pwFileContent)
+	if err != nil {
+		return err
+	}
+	pwFile.Close()
+	return nil
+}

pkg/s3/mounter_s3ql.go

@@ -0,0 +1,117 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"gopkg.in/ini.v1"
+)
+
+// Implements Mounter
+type s3qlMounter struct {
+	url        string
+	bucketURL  string
+	login      string
+	password   string
+	passphrase string
+	options    []string
+	ssl        bool
+	targetPath string
+}
+
+const (
+	s3qlCmdMkfs    = "mkfs.s3ql"
+	s3qlCmdMount   = "mount.s3ql"
+	s3qlCmdUnmount = "umount.s3ql"
+)
+
+func newS3qlMounter(bucket string, cfg *Config) (Mounter, error) {
+	url, err := url.Parse(cfg.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	ssl := url.Scheme != "http"
+	if strings.Contains(url.Scheme, "http") {
+		url.Scheme = "s3c"
+	}
+	s3ql := &s3qlMounter{
+		url:        url.String(),
+		login:      cfg.AccessKeyID,
+		password:   cfg.SecretAccessKey,
+		passphrase: cfg.EncryptionKey,
+		ssl:        ssl,
+	}
+
+	url.Path = path.Join(url.Path, bucket)
+	s3ql.bucketURL = url.String()
+
+	if !ssl {
+		s3ql.options = []string{"--backend-options", "no-ssl"}
+	}
+
+	return s3ql, s3ql.writeConfig()
+}
+
+func (s3ql *s3qlMounter) Format() error {
+	// force creation to ignore existing data
+	args := []string{
+		s3ql.bucketURL,
+		"--force",
+	}
+
+	p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase)
+	reader := bytes.NewReader([]byte(p))
+	return s3qlCmd(s3qlCmdMkfs, append(args, s3ql.options...), reader)
+}
+
+func (s3ql *s3qlMounter) Mount(targetPath string) error {
+	args := []string{
+		s3ql.bucketURL,
+		targetPath,
+		"--allow-other",
+	}
+	return s3qlCmd(s3qlCmdMount, append(args, s3ql.options...), nil)
+}
+
+func (s3ql *s3qlMounter) Unmount(targetPath string) error {
+	return s3qlCmd(s3qlCmdUnmount, []string{targetPath}, nil)
+}
+
+func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error {
+	cmd := exec.Command(s3qlCmd, args...)
+	if stdin != nil {
+		cmd.Stdin = stdin
+	}
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Error running s3ql command: %s", out)
+	}
+	return nil
+}
+
+func (s3ql *s3qlMounter) writeConfig() error {
+	s3qlIni := ini.Empty()
+	section, err := s3qlIni.NewSection("s3ql")
+	if err != nil {
+		return err
+	}
+
+	section.NewKey("storage-url", s3ql.url)
+	section.NewKey("backend-login", s3ql.login)
+	section.NewKey("backend-password", s3ql.password)
+	section.NewKey("fs-passphrase", s3ql.passphrase)
+
+	authDir := os.Getenv("HOME") + "/.s3ql"
+	authFile := authDir + "/authinfo2"
+	os.Mkdir(authDir, 0700)
+	s3qlIni.SaveTo(authFile)
+	os.Chmod(authFile, 0600)
+	return nil
+}

pkg/s3/nodeserver.go

@@ -17,7 +17,6 @@ limitations under the License.
 package s3
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/golang/glog"
@@ -31,12 +30,6 @@ import (
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 )
 
-const (
-	mounterKey    = "mounter"
-	s3fsMounter   = "s3fs"
-	goofysMounter = "goofys"
-)
-
 type nodeServer struct {
 	*csicommon.DefaultNodeServer
 	*s3
@@ -85,18 +78,13 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n",
 		targetPath, deviceID, readOnly, volumeID, attrib, mountFlags)
 
-	mounter, exists := attrib[mounterKey]
+	mounter, err := newMounter(volumeID, ns.s3.cfg)
-	if !exists || mounter == s3fsMounter {
+	if err != nil {
-		if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil {
 		return nil, err
 	}
-	} else if mounter == goofysMounter {
+	if err := mounter.Mount(targetPath); err != nil {
-		if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil {
 		return nil, err
 	}
-	} else {
-		return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter)
-	}
 
 	glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath)
 
@@ -113,8 +101,11 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
 	}
 
-	err := mount.New("").Unmount(req.GetTargetPath())
+	mounter, err := newMounter(req.GetVolumeId(), ns.s3.cfg)
 	if err != nil {
+		return nil, err
+	}
+	if err := mounter.Unmount(req.GetTargetPath()); err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 	glog.V(4).Infof("s3: bucket %s has been unmounted.", req.GetVolumeId())
@@ -122,28 +113,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
 
-func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+func (ns *nodeServer) NodeStageVolume(
-
+	ctx context.Context,
-	// Check arguments
+	req *csi.NodeStageVolumeRequest) (
-	if len(req.GetVolumeId()) == 0 {
+	*csi.NodeStageVolumeResponse, error) {
-		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
+	return nil, status.Error(codes.Unimplemented, "")
-	}
-	if len(req.GetStagingTargetPath()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
 }
 
-	return &csi.NodeStageVolumeResponse{}, nil
+func (ns *nodeServer) NodeUnstageVolume(
-}
+	ctx context.Context,
-
+	req *csi.NodeUnstageVolumeRequest) (
-func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	*csi.NodeUnstageVolumeResponse, error) {
-
+	return nil, status.Error(codes.Unimplemented, "")
-	// Check arguments
-	if len(req.GetVolumeId()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
-	}
-	if len(req.GetStagingTargetPath()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
-	}
-
-	return &csi.NodeUnstageVolumeResponse{}, nil
 }

pkg/s3/s3-client.go

@@ -17,10 +17,6 @@ type s3Client struct {
 	minio *minio.Client
 }
 
-type bucketMetadata struct {
-	CapacityBytes int64
-}
-
 func newS3Client(cfg *Config) (*s3Client, error) {
 	var client = &s3Client{}
 

pkg/s3/s3-driver_suite_test.go

@@ -0,0 +1,116 @@
+package s3_test
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/ctrox/csi-s3-driver/pkg/s3"
+	"github.com/kubernetes-csi/csi-test/pkg/sanity"
+)
+
+const ()
+
+var _ = Describe("S3Driver", func() {
+	mntDir, err := ioutil.TempDir("", "mnt")
+	if err != nil {
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	AfterSuite(func() {
+		os.RemoveAll(mntDir)
+	})
+
+	Context("goofys", func() {
+		socket := "/tmp/csi-goofys.sock"
+		csiEndpoint := "unix://" + socket
+		cfg := &s3.Config{
+			AccessKeyID:     "FJDSJ",
+			SecretAccessKey: "DSG643HGDS",
+			Endpoint:        "http://127.0.0.1:9000",
+			Mounter:         "goofys",
+		}
+		if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		driver, err := s3.NewS3("test-node", csiEndpoint, cfg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		go driver.Run()
+
+		Describe("CSI sanity", func() {
+			sanityCfg := &sanity.Config{
+				TargetPath:     mntDir,
+				Address:        csiEndpoint,
+				TestVolumeSize: 1,
+			}
+			sanity.GinkgoTest(sanityCfg)
+		})
+	})
+
+	Context("s3fs", func() {
+		socket := "/tmp/csi-s3fs.sock"
+		csiEndpoint := "unix://" + socket
+		cfg := &s3.Config{
+			AccessKeyID:     "FJDSJ",
+			SecretAccessKey: "DSG643HGDS",
+			Endpoint:        "http://127.0.0.1:9000",
+			Mounter:         "s3fs",
+		}
+		if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		driver, err := s3.NewS3("test-node", csiEndpoint, cfg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		go driver.Run()
+
+		defer os.RemoveAll(mntDir)
+
+		Describe("CSI sanity", func() {
+			sanityCfg := &sanity.Config{
+				TargetPath:     mntDir,
+				Address:        csiEndpoint,
+				TestVolumeSize: 1,
+			}
+			sanity.GinkgoTest(sanityCfg)
+		})
+	})
+
+	Context("s3ql", func() {
+		socket := "/tmp/csi-s3ql.sock"
+		csiEndpoint := "unix://" + socket
+
+		cfg := &s3.Config{
+			AccessKeyID:     "FJDSJ",
+			SecretAccessKey: "DSG643HGDS",
+			Endpoint:        "http://127.0.0.1:9000",
+			Mounter:         "s3ql",
+		}
+		if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
+			Expect(err).NotTo(HaveOccurred())
+		}
+		driver, err := s3.NewS3("test-node", csiEndpoint, cfg)
+		if err != nil {
+			log.Fatal(err)
+		}
+		go driver.Run()
+
+		defer os.RemoveAll(mntDir)
+
+		Describe("CSI sanity", func() {
+			sanityCfg := &sanity.Config{
+				TargetPath:     mntDir,
+				Address:        csiEndpoint,
+				TestVolumeSize: 1,
+			}
+			sanity.GinkgoTest(sanityCfg)
+		})
+	})
+
+})

pkg/s3/s3-driver_test.go

@@ -17,42 +17,13 @@ limitations under the License.
 package s3
 
 import (
-	"io/ioutil"
-	"log"
-	"os"
 	"testing"
 
-	"github.com/kubernetes-csi/csi-test/pkg/sanity"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
-func TestDriver(t *testing.T) {
+func TestS3Driver(t *testing.T) {
-	socket := "/tmp/csi.sock"
+	RegisterFailHandler(Fail)
-	endpoint := "unix://" + socket
+	RunSpecs(t, "S3Driver")
-
-	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
-		t.Fatalf("failed to remove unix domain socket file %s, error: %s", socket, err)
-	}
-	cfg := &Config{
-		AccessKeyID:     "FJDSJ",
-		SecretAccessKey: "DSG643HGDS",
-		Endpoint:        "http://127.0.0.1:9000",
-	}
-	driver, err := NewS3("test-node", endpoint, cfg)
-	if err != nil {
-		log.Fatal(err)
-	}
-	go driver.Run()
-
-	mntDir, err := ioutil.TempDir("", "mnt")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(mntDir)
-
-	sanityCfg := &sanity.Config{
-		TargetPath: mntDir,
-		Address:    endpoint,
-	}
-
-	sanity.Test(t, sanityCfg)
 }

pkg/s3/s3fs.go

@@ -1,43 +0,0 @@
-package s3
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-)
-
-func s3fsMount(bucket string, cfg *Config, targetPath string) error {
-	if err := writes3fsPass(cfg); err != nil {
-		return err
-	}
-	args := []string{
-		fmt.Sprintf("%s", bucket),
-		fmt.Sprintf("%s", targetPath),
-		"-o", "sigv2",
-		"-o", "use_path_request_style",
-		"-o", fmt.Sprintf("url=%s", cfg.Endpoint),
-		"-o", fmt.Sprintf("endpoint=%s", cfg.Region),
-		"-o", "allow_other",
-		"-o", "mp_umask=000",
-	}
-	cmd := exec.Command("s3fs", args...)
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("Error mounting using s3fs, output: %s", out)
-	}
-	return nil
-}
-
-func writes3fsPass(cfg *Config) error {
-	pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME"))
-	pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600)
-	if err != nil {
-		return err
-	}
-	_, err = pwFile.WriteString(cfg.AccessKeyID + ":" + cfg.SecretAccessKey)
-	if err != nil {
-		return err
-	}
-	pwFile.Close()
-	return nil
-}

test/Dockerfile

@@ -1,8 +1,24 @@
 FROM golang:stretch
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
 LABEL description="s3 fuse csi plugin"
+ARG S3QL_VERSION=release-2.28
+
+RUN apt-get update && \
+    apt-get install -y \
+      s3fs wget psmisc procps python3 python3-setuptools \
+      python3-dev python3-pip python3-llfuse pkg-config \
+      sqlite3 libsqlite3-dev python3-apsw cython && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install defusedxml dugong requests pycrypto
+
+WORKDIR /usr/src
+RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
+RUN tar -xzf ${S3QL_VERSION}.tar.gz
+WORKDIR /usr/src/s3ql-${S3QL_VERSION}
+RUN python3 setup.py build_cython build_ext --inplace
+RUN python3 setup.py install
 
-RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/*
 RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd
 
 CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"]