Add experimental s3ql mounter

Gopkg.lock

@@ -334,6 +334,12 @@
   revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
   version = "v1.13.0"
 
+[[projects]]
+  name = "gopkg.in/ini.v1"
+  packages = ["."]
+  revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
+  version = "v1.38.1"
+
 [[projects]]
   name = "gopkg.in/yaml.v2"
   packages = ["."]
@@ -366,6 +372,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4"
+  inputs-digest = "9bd4175acb8ce47fe57c3d859bc6b061a6c5dd3017f57777f3fefb27d0020d75"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -37,3 +37,7 @@
 [prune]
   go-tests = true
   unused-packages = true
+
+[[constraint]]
+  name = "gopkg.in/ini.v1"
+  version = "1.38.1"

Makefile

@@ -16,7 +16,7 @@
 PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver
 REGISTRY_NAME=ctrox
 IMAGE_NAME=csi-s3-driver
-IMAGE_VERSION=0.1.0
+IMAGE_VERSION=0.2.0
 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
 TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test
 
@@ -27,7 +27,7 @@ test:
 	docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile .
 	docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG)
 container: build
-	docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile .
+	docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql .
 push: container
 	docker push $(IMAGE_TAG)
 clean:

cmd/s3driver/Dockerfile.s3ql

@@ -0,0 +1,23 @@
+FROM debian:stretch
+LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
+LABEL description="s3 fuse csi plugin"
+ARG S3QL_VERSION=release-2.28
+
+RUN apt-get update && \
+    apt-get install -y \
+      s3fs wget python3 python3-setuptools \
+      python3-dev python3-pip python3-llfuse pkg-config \
+      sqlite3 libsqlite3-dev python3-apsw cython && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install defusedxml dugong requests pycrypto
+
+WORKDIR /usr/src
+RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
+RUN tar -xzf ${S3QL_VERSION}.tar.gz
+WORKDIR /usr/src/s3ql-${S3QL_VERSION}
+RUN python3 setup.py build_cython build_ext --inplace
+RUN python3 setup.py install
+
+COPY ./_output/s3driver /s3driver
+ENTRYPOINT ["/s3driver"]

cmd/s3driver/main.go

@@ -35,6 +35,7 @@ var (
 	secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use")
 	s3endpoint      = flag.String("s3-endpoint", "", "S3 Endpoint URL to use")
 	region          = flag.String("region", "", "S3 Region to use")
+	encryptionKey   = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)")
 )
 
 func main() {
@@ -45,6 +46,7 @@ func main() {
 		SecretAccessKey: *secretAccessKey,
 		Endpoint:        *s3endpoint,
 		Region:          *region,
+		EncryptionKey:   *encryptionKey,
 	}
 
 	driver, err := s3.NewS3(*nodeID, *endpoint, cfg)

deploy/kubernetes/csi-s3-driver.yaml

@@ -75,7 +75,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: ctrox/csi-s3-driver:0.1.0
+          image: ctrox/csi-s3-driver:0.2.0
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(NODE_ID)"
@@ -83,6 +83,7 @@ spec:
             - "--secret-access-key=$(SECRET_ACCESS_KEY)"
             - "--s3-endpoint=$(S3_ENDPOINT)"
             - "--region=$(REGION)"
+            - "--encryption-key=$(ENCRYPTION_KEY)"
             - "--v=4"
           env:
             - name: CSI_ENDPOINT
@@ -111,6 +112,11 @@ spec:
                 secretKeyRef:
                   name: csi-s3-secret
                   key: region
+            - name: ENCRYPTION_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: encryptionKey
           imagePullPolicy: "Always"
           volumeMounts:
             - name: plugin-dir

deploy/kubernetes/provisioner.yaml

@@ -79,7 +79,7 @@ spec:
             - name: socket-dir
               mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver
         - name: s3-csi-driver
-          image: ctrox/csi-s3-driver:0.1.0
+          image: ctrox/csi-s3-driver:0.2.0
           args:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(NODE_ID)"
@@ -87,6 +87,7 @@ spec:
             - "--secret-access-key=$(SECRET_ACCESS_KEY)"
             - "--s3-endpoint=$(S3_ENDPOINT)"
             - "--region=$(REGION)"
+            - "--encryption-key=$(ENCRYPTION_KEY)"
             - "--v=4"
           env:
             - name: CSI_ENDPOINT
@@ -115,6 +116,11 @@ spec:
                 secretKeyRef:
                   name: csi-s3-secret
                   key: region
+            - name: ENCRYPTION_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: csi-s3-secret
+                  key: encryptionKey
           imagePullPolicy: "Always"
           volumeMounts:
             - name: socket-dir

deploy/kubernetes/pvc.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: default
 spec:
   accessModes:
-  - ReadWriteMany
+  - ReadWriteOnce
   resources:
     requests:
       storage: 5Gi

deploy/kubernetes/secret.yaml

@@ -8,3 +8,5 @@ stringData:
   endpoint: <S3_ENDPOINT_URL>
   # If not on S3, just set it to ""
   region: <S3_REGION>
+  # Currently only for s3ql
+  # encryptionKey: <FS encryption key>

deploy/kubernetes/storageclass.yaml

@@ -6,6 +6,6 @@ metadata:
 provisioner: ch.ctrox.csi.s3-driver
 parameters:
   # specify which mounter to use
-  # can be set to s3fs or goofys
+  # can be set to s3fs, goofys or s3ql
   # s3fs is the default
-  # mounter: s3fs
+  # mounter: s3ql

pkg/s3/config.go

@@ -6,4 +6,6 @@ type Config struct {
 	SecretAccessKey string
 	Region          string
 	Endpoint        string
+	Mounter         string
+	EncryptionKey   string
 }

pkg/s3/controllerserver.go

@@ -60,6 +60,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 			return nil, err
 		}
 	}
+	mounter := cs.s3.cfg.Mounter
+	if mounter == "" {
+		mounter = req.GetParameters()[mounterKey]
+	}
+	switch mounter {
+	case s3qlMounter:
+		if err := s3qlCreate(volumeID, cs.s3.cfg); err != nil {
+			return nil, err
+		}
+	}
 
 	glog.V(4).Infof("create volume %s", volumeID)
 	s3Vol := s3Volume{}
@@ -123,6 +133,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
 		return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId()))
 	}
 
-	// We currently support all capabilities
+	for _, cap := range req.VolumeCapabilities {
-	return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil
+		if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
+			return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
+		}
+	}
+	return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil
 }

pkg/s3/nodeserver.go

@@ -35,6 +35,7 @@ const (
 	mounterKey    = "mounter"
 	s3fsMounter   = "s3fs"
 	goofysMounter = "goofys"
+	s3qlMounter   = "s3ql"
 )
 
 type nodeServer struct {
@@ -85,17 +86,26 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n",
 		targetPath, deviceID, readOnly, volumeID, attrib, mountFlags)
 
-	mounter, exists := attrib[mounterKey]
+	mounter := ns.s3.cfg.Mounter
-	if !exists || mounter == s3fsMounter {
+	if mounter == "" {
+		mounter = attrib[mounterKey]
+	}
+	switch mounter {
+	case "":
+	case s3fsMounter:
 		if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil {
 			return nil, err
 		}
-	} else if mounter == goofysMounter {
+	case goofysMounter:
 		if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil {
 			return nil, err
 		}
-	} else {
+	case s3qlMounter:
-		return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter)
+		if err := s3qlMount(volumeID, ns.s3.cfg, targetPath); err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, ns.s3.cfg.Mounter)
 	}
 
 	glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath)
@@ -121,29 +131,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
-
+func (ns *nodeServer) NodeStageVolume(
-func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
+	ctx context.Context,
-
+	req *csi.NodeStageVolumeRequest) (
-	// Check arguments
+	*csi.NodeStageVolumeResponse, error) {
-	if len(req.GetVolumeId()) == 0 {
+	return nil, status.Error(codes.Unimplemented, "")
-		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
-	}
-	if len(req.GetStagingTargetPath()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
 }
 
-	return &csi.NodeStageVolumeResponse{}, nil
+func (ns *nodeServer) NodeUnstageVolume(
-}
+	ctx context.Context,
-
+	req *csi.NodeUnstageVolumeRequest) (
-func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
+	*csi.NodeUnstageVolumeResponse, error) {
-
+	return nil, status.Error(codes.Unimplemented, "")
-	// Check arguments
-	if len(req.GetVolumeId()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
-	}
-	if len(req.GetStagingTargetPath()) == 0 {
-		return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
-	}
-
-	return &csi.NodeUnstageVolumeResponse{}, nil
 }

pkg/s3/s3-client.go

@@ -17,10 +17,6 @@ type s3Client struct {
 	minio *minio.Client
 }
 
-type bucketMetadata struct {
-	CapacityBytes int64
-}
-
 func newS3Client(cfg *Config) (*s3Client, error) {
 	var client = &s3Client{}
 

pkg/s3/s3-driver_test.go

@@ -36,6 +36,7 @@ func TestDriver(t *testing.T) {
 		AccessKeyID:     "FJDSJ",
 		SecretAccessKey: "DSG643HGDS",
 		Endpoint:        "http://127.0.0.1:9000",
+		EncryptionKey:   "IskEwCuEg6drywi",
 	}
 	driver, err := NewS3("test-node", endpoint, cfg)
 	if err != nil {

pkg/s3/s3ql.go

@@ -0,0 +1,129 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"gopkg.in/ini.v1"
+)
+
+type s3fsConfig struct {
+	url        string
+	bucketURL  string
+	login      string
+	password   string
+	passphrase string
+	options    []string
+	ssl        bool
+	targetPath string
+}
+
+const (
+	s3qlCmdMkfs  = "mkfs.s3ql"
+	s3qlCmdMount = "mount.s3ql"
+)
+
+func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) {
+	url, err := url.Parse(cfg.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	ssl := url.Scheme != "http"
+	if strings.Contains(url.Scheme, "http") {
+		url.Scheme = "s3c"
+	}
+	s3ql := &s3fsConfig{
+		url:        url.String(),
+		login:      cfg.AccessKeyID,
+		password:   cfg.SecretAccessKey,
+		passphrase: cfg.EncryptionKey,
+		ssl:        ssl,
+		targetPath: targetPath,
+	}
+
+	url.Path = path.Join(url.Path, bucket)
+	s3ql.bucketURL = url.String()
+
+	if !ssl {
+		s3ql.options = []string{"--backend-options", "no-ssl"}
+	}
+
+	return s3ql, s3ql.writeConfig()
+}
+
+func s3qlCreate(bucket string, cfg *Config) error {
+	s3ql, err := newS3ql(bucket, "unknown", cfg)
+	if err != nil {
+		return err
+	}
+	return s3ql.create()
+}
+
+func s3qlMount(bucket string, cfg *Config, targetPath string) error {
+	s3ql, err := newS3ql(bucket, targetPath, cfg)
+	if err != nil {
+		return err
+	}
+
+	return s3ql.mount()
+}
+
+func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error {
+	cmd := exec.Command(s3qlCmd, args...)
+	if stdin != nil {
+		cmd.Stdin = stdin
+	}
+
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Error running s3ql command: %s", out)
+	}
+	return nil
+}
+
+func (cfg *s3fsConfig) create() error {
+	// force creation to ignore existing data
+	args := []string{
+		cfg.bucketURL,
+		"--force",
+	}
+
+	p := fmt.Sprintf("%s\n%s\n", cfg.passphrase, cfg.passphrase)
+	reader := bytes.NewReader([]byte(p))
+	return s3qlCmd(s3qlCmdMkfs, append(args, cfg.options...), reader)
+}
+
+func (cfg *s3fsConfig) mount() error {
+	args := []string{
+		cfg.bucketURL,
+		cfg.targetPath,
+		"--allow-other",
+	}
+	return s3qlCmd(s3qlCmdMount, append(args, cfg.options...), nil)
+}
+
+func (cfg *s3fsConfig) writeConfig() error {
+	s3qlIni := ini.Empty()
+	section, err := s3qlIni.NewSection("s3ql")
+	if err != nil {
+		return err
+	}
+
+	section.NewKey("storage-url", cfg.url)
+	section.NewKey("backend-login", cfg.login)
+	section.NewKey("backend-password", cfg.password)
+	section.NewKey("fs-passphrase", cfg.passphrase)
+
+	authDir := os.Getenv("HOME") + "/.s3ql"
+	authFile := authDir + "/authinfo2"
+	os.Mkdir(authDir, 0700)
+	s3qlIni.SaveTo(authFile)
+	os.Chmod(authFile, 0600)
+	return nil
+}

test/Dockerfile

@@ -1,8 +1,24 @@
 FROM golang:stretch
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
 LABEL description="s3 fuse csi plugin"
+ARG S3QL_VERSION=release-2.28
+
+RUN apt-get update && \
+    apt-get install -y \
+      s3fs wget python3 python3-setuptools \
+      python3-dev python3-pip python3-llfuse pkg-config \
+      sqlite3 libsqlite3-dev python3-apsw cython && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install defusedxml dugong requests pycrypto
+
+WORKDIR /usr/src
+RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
+RUN tar -xzf ${S3QL_VERSION}.tar.gz
+WORKDIR /usr/src/s3ql-${S3QL_VERSION}
+RUN python3 setup.py build_cython build_ext --inplace
+RUN python3 setup.py install
 
-RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/*
 RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd
 
 CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"]