Remove s3ql

s3ql does not work very well in a dynamic environment like k8s.
Also as it needs a ton of dependencies just to get it built makes
it hard to maintain.

Makefile

@@ -16,8 +16,9 @@
 PROJECT_DIR=/app
 REGISTRY_NAME=ctrox
 IMAGE_NAME=csi-s3
-IMAGE_VERSION=1.0.1-alpha
+VERSION ?= dev
-IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
+IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION)
+FULL_IMAGE_TAG=$(IMAGE_TAG)-full
 TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test
 
 build:
@@ -27,8 +28,10 @@ test:
 	docker run --rm --privileged -v $(PWD):$(PROJECT_DIR) --device /dev/fuse $(TEST_IMAGE_TAG)
 container: build
 	docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile .
+	docker build -t $(FULL_IMAGE_TAG) --build-arg VERSION=$(VERSION) -f cmd/s3driver/Dockerfile.full .
 push: container
 	docker push $(IMAGE_TAG)
+	docker push $(FULL_IMAGE_TAG)
 clean:
 	go clean -r -x
 	-rm -rf _output

README.md

@@ -10,14 +10,12 @@ This is still very experimental and should not be used in any production environ
 
 ### Requirements
 
-* Kubernetes 1.13+
+* Kubernetes 1.13+ (CSI v1.0.0 compatibility)
 * Kubernetes has to allow privileged containers
 * Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`)
 
 ### 1. Create a secret with your S3 credentials
 
-The endpoint is optional if you are using something else than AWS S3. Also the region can be empty if you are using some other S3 compatible storage.
-
 ```yaml
 apiVersion: v1
 kind: Secret
@@ -30,11 +28,10 @@ stringData:
   endpoint: <S3_ENDPOINT_URL>
   # If not on S3, set it to ""
   region: <S3_REGION>
-  # Currently only for s3ql
-  # If not using s3ql, set it to ""
-  encryptionKey: <FS_ENCRYPTION_KEY>
 ```
 
+The region can be empty if you are using some other S3 compatible storage.
+
 ### 2. Deploy the driver
 
 ```bash
@@ -63,7 +60,7 @@ kubectl create -f pvc.yaml
 ```bash
 $ kubectl get pvc csi-s3-pvc
 NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
-csi-s3-pvc   Bound     pvc-c5d4634f-8507-11e8-9f33-0e243832354b   5Gi        RWX            csi-s3         9s
+csi-s3-pvc   Bound     pvc-c5d4634f-8507-11e8-9f33-0e243832354b   5Gi        RWO            csi-s3         9s
 ```
 
 * Create a test pod which mounts your volume:
@@ -96,7 +93,6 @@ The driver can be configured to use one of these mounters to mount buckets:
 * [rclone](https://rclone.org/commands/rclone_mount)
 * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse)
 * [goofys](https://github.com/kahing/goofys)
-* [s3ql](https://github.com/s3ql/s3ql)
 * [s3backer](https://github.com/archiecobbs/s3backer)
 
 The mounter can be set as a parameter in the storage class. You can also create multiple storage classes for each mounter if you like.
@@ -121,16 +117,7 @@ All mounters have different strengths and weaknesses depending on your use case.
 * Files can be viewed normally with any S3 client
 * Does not support appends or random writes
 
-#### s3ql (not recommended*)
+#### s3backer (experimental*)
-
-* (Almost) full POSIX compatibility
-* Uses its own object format
-* Files are not readable with other S3 clients
-* Support appends
-* Supports compression before upload
-* Supports encryption before upload
-
-#### s3backer (not recommended*)
 
 * Represents a block device stored on S3
 * Allows to use a real filesystem
@@ -139,7 +126,8 @@ All mounters have different strengths and weaknesses depending on your use case.
 * Supports compression before upload (Not yet implemented in this driver)
 * Supports encryption before upload (Not yet implemented in this driver)
 
-*s3ql and s3backer are not recommended at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod.
+*s3backer is experimental at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod.
+The s3backer binary is not bundled with the normal docker image to keep that as small as possible. Use the `<version>-full` image tag for testing s3backer.
 
 Fore more detailed limitations consult the documentation of the different projects.
 

cmd/s3driver/Dockerfile

@@ -2,15 +2,16 @@ FROM debian:stretch
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
 LABEL description="csi-s3 slim image"
 
+# s3fs and some other dependencies
 RUN apt-get update && \
   apt-get install -y \
-  s3fs wget unzip && \
+  s3fs curl unzip && \
   rm -rf /var/lib/apt/lists/*
 
 # install rclone
-ARG RCLONE_VERSION=v1.46
+ARG RCLONE_VERSION=v1.47.0
 RUN cd /tmp \
-  && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
+  && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
   && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
   && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
   && rm -r /tmp/rclone*

cmd/s3driver/Dockerfile.full

@@ -1,16 +1,3 @@
-FROM python:3.6 as s3ql-deps
-LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
-LABEL description="s3ql dependencies"
-
-RUN apt-get update && \
-    apt-get install -y \
-       python3 python3-setuptools \
-      python3-dev python3-pip pkg-config cython \
-      libfuse-dev libattr1-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto
-
 FROM debian:stretch as s3backer
 ARG S3BACKER_VERSION=1.5.0
 
@@ -33,36 +20,29 @@ WORKDIR /src/s3backer
 RUN git checkout tags/${S3BACKER_VERSION}
 
 RUN ./autogen.sh && \
-    ./configure && \
+  ./configure && \
-    make && \
+  make && \
-    make install
+  make install
 
-FROM python:3.6-slim
+FROM debian:stretch
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
-LABEL description="csi-s3 full image"
+LABEL description="csi-s3 image"
-
-RUN apt-get update && \
-    apt-get install -y \
-      libfuse2 gcc sqlite3 libsqlite3-dev \
-      s3fs psmisc procps libcurl3 xfsprogs wget unzip && \
-    rm -rf /var/lib/apt/lists/*
-
-ARG S3QL_VERSION=2.29
-ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2
-
-COPY --from=s3ql-deps /root/.cache /root/.cache
-COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
-RUN pip install ${S3QL_URL} && rm -rf /root/.cache
-
 COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer
 
+# s3fs and some other dependencies
+RUN apt-get update && \
+  apt-get install -y \
+  libfuse2 gcc sqlite3 libsqlite3-dev \
+  s3fs psmisc procps libcurl3 xfsprogs curl unzip && \
+  rm -rf /var/lib/apt/lists/*
+
 # install rclone
-ARG RCLONE_VERSION=v1.46
+ARG RCLONE_VERSION=v1.47.0
 RUN cd /tmp \
-    && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
+  && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
-    && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
+  && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
-    && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
+  && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
-    && rm -r /tmp/rclone*
+  && rm -r /tmp/rclone*
 
 COPY ./_output/s3driver /s3driver
 ENTRYPOINT ["/s3driver"]

deploy/kubernetes/secret.yaml

@@ -9,6 +9,3 @@ stringData:
   endpoint: https://s3.eu-central-1.amazonaws.com
   # If not on S3, set it to ""
   region: <S3_REGION>
-  # Currently only for s3ql
-  # If not using s3ql, set it to ""
-  encryptionKey: ""

deploy/kubernetes/storageclass.yaml

@@ -6,7 +6,7 @@ metadata:
 provisioner: ch.ctrox.csi.s3-driver
 parameters:
   # specify which mounter to use
-  # can be set to s3backer, s3ql, s3fs or goofys
+  # can be set to rclone, s3fs, goofys or s3backer
   mounter: rclone
   csi.storage.k8s.io/provisioner-secret-name: csi-s3-secret
   csi.storage.k8s.io/provisioner-secret-namespace: kube-system

pkg/s3/mounter.go

@@ -20,7 +20,6 @@ type Mounter interface {
 const (
 	s3fsMounterType     = "s3fs"
 	goofysMounterType   = "goofys"
-	s3qlMounterType     = "s3ql"
 	s3backerMounterType = "s3backer"
 	rcloneMounterType   = "rclone"
 	mounterTypeKey      = "mounter"
@@ -40,9 +39,6 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) {
 	case goofysMounterType:
 		return newGoofysMounter(bucket, cfg)
 
-	case s3qlMounterType:
-		return newS3qlMounter(bucket, cfg)
-
 	case s3backerMounterType:
 		return newS3backerMounter(bucket, cfg)
 

pkg/s3/mounter_s3ql.go

@@ -1,114 +0,0 @@
-package s3
-
-import (
-	"bytes"
-	"fmt"
-	"net/url"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	"gopkg.in/ini.v1"
-)
-
-// Implements Mounter
-type s3qlMounter struct {
-	bucket     *bucket
-	url        string
-	bucketURL  string
-	login      string
-	password   string
-	passphrase string
-	options    []string
-	ssl        bool
-	targetPath string
-}
-
-const (
-	s3qlCmdMkfs    = "mkfs.s3ql"
-	s3qlCmdMount   = "mount.s3ql"
-	s3qlCmdUnmount = "umount.s3ql"
-)
-
-func newS3qlMounter(b *bucket, cfg *Config) (Mounter, error) {
-	url, err := url.Parse(cfg.Endpoint)
-	if err != nil {
-		return nil, err
-	}
-	ssl := url.Scheme != "http"
-	if strings.Contains(url.Scheme, "http") {
-		url.Scheme = "s3c"
-	}
-	s3ql := &s3qlMounter{
-		bucket:     b,
-		url:        url.String(),
-		login:      cfg.AccessKeyID,
-		password:   cfg.SecretAccessKey,
-		passphrase: cfg.EncryptionKey,
-		ssl:        ssl,
-	}
-
-	// s3ql requires a trailing slash or it will just
-	// prepend the fspath to the s3ql files
-	url.Path = path.Join(url.Path, b.Name, b.FSPath) + "/"
-	s3ql.bucketURL = url.String()
-
-	if !ssl {
-		s3ql.options = []string{"--backend-options", "no-ssl"}
-	}
-
-	return s3ql, s3ql.writeConfig()
-}
-
-func (s3ql *s3qlMounter) Stage(stagePath string) error {
-	// force creation to ignore existing data
-	args := []string{
-		s3ql.bucketURL,
-		"--force",
-	}
-
-	p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase)
-	reader := bytes.NewReader([]byte(p))
-	cmd := exec.Command(s3qlCmdMkfs, append(args, s3ql.options...)...)
-	cmd.Stdin = reader
-
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return fmt.Errorf("Error running s3ql command: %s", out)
-	}
-	return nil
-}
-
-func (s3ql *s3qlMounter) Unstage(stagePath string) error {
-	return nil
-}
-
-func (s3ql *s3qlMounter) Mount(source string, target string) error {
-	args := []string{
-		s3ql.bucketURL,
-		target,
-		"--allow-other",
-	}
-	return fuseMount(target, s3qlCmdMount, append(args, s3ql.options...))
-}
-
-func (s3ql *s3qlMounter) writeConfig() error {
-	s3qlIni := ini.Empty()
-	section, err := s3qlIni.NewSection("s3ql")
-	if err != nil {
-		return err
-	}
-
-	section.NewKey("storage-url", s3ql.url)
-	section.NewKey("backend-login", s3ql.login)
-	section.NewKey("backend-password", s3ql.password)
-	section.NewKey("fs-passphrase", s3ql.passphrase)
-
-	authDir := os.Getenv("HOME") + "/.s3ql"
-	authFile := authDir + "/authinfo2"
-	os.Mkdir(authDir, 0700)
-	s3qlIni.SaveTo(authFile)
-	os.Chmod(authFile, 0600)
-	return nil
-}

pkg/s3/s3-driver_suite_test.go

@@ -65,33 +65,6 @@ var _ = Describe("S3Driver", func() {
 		})
 	})
 
-	Context("s3ql", func() {
-		socket := "/tmp/csi-s3ql.sock"
-		csiEndpoint := "unix://" + socket
-
-		if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
-			Expect(err).NotTo(HaveOccurred())
-		}
-		driver, err := s3.NewS3("test-node", csiEndpoint)
-		if err != nil {
-			log.Fatal(err)
-		}
-		go driver.Run()
-
-		Describe("CSI sanity", func() {
-			sanityCfg := &sanity.Config{
-				TargetPath:  os.TempDir() + "/s3ql-target",
-				StagingPath: os.TempDir() + "/s3ql-staging",
-				Address:     csiEndpoint,
-				SecretsFile: "../../test/secret.yaml",
-				TestVolumeParameters: map[string]string{
-					"mounter": "s3ql",
-				},
-			}
-			sanity.GinkgoTest(sanityCfg)
-		})
-	})
-
 	Context("s3backer", func() {
 		socket := "/tmp/csi-s3backer.sock"
 		csiEndpoint := "unix://" + socket

test/Dockerfile

@@ -1,4 +1,4 @@
-FROM ctrox/csi-s3:dev
+FROM ctrox/csi-s3:dev-full
 LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
 LABEL description="csi-s3 testing image"