From f80104f215d46b621328113b50bb6b325f34bb91 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Wed, 15 May 2019 20:42:50 +0200 Subject: [PATCH] Remove s3ql s3ql does not work very well in a dynamic environment like k8s. Also as it needs a ton of dependencies just to get it built makes it hard to maintain. --- Makefile | 7 +- README.md | 26 ++----- cmd/s3driver/Dockerfile | 7 +- cmd/s3driver/Dockerfile.full | 54 +++++-------- deploy/kubernetes/secret.yaml | 3 - deploy/kubernetes/storageclass.yaml | 2 +- pkg/s3/mounter.go | 4 - pkg/s3/mounter_s3ql.go | 114 ---------------------------- pkg/s3/s3-driver_suite_test.go | 27 ------- test/Dockerfile | 2 +- 10 files changed, 35 insertions(+), 211 deletions(-) delete mode 100644 pkg/s3/mounter_s3ql.go diff --git a/Makefile b/Makefile index ab03abf..91a2059 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,9 @@ PROJECT_DIR=/app REGISTRY_NAME=ctrox IMAGE_NAME=csi-s3 -IMAGE_VERSION=1.0.1-alpha -IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) +VERSION ?= dev +IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION) +FULL_IMAGE_TAG=$(IMAGE_TAG)-full TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test build: @@ -27,8 +28,10 @@ test: docker run --rm --privileged -v $(PWD):$(PROJECT_DIR) --device /dev/fuse $(TEST_IMAGE_TAG) container: build docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile . + docker build -t $(FULL_IMAGE_TAG) --build-arg VERSION=$(VERSION) -f cmd/s3driver/Dockerfile.full . push: container docker push $(IMAGE_TAG) + docker push $(FULL_IMAGE_TAG) clean: go clean -r -x -rm -rf _output diff --git a/README.md b/README.md index 4d57413..f36208f 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,12 @@ This is still very experimental and should not be used in any production environ ### Requirements -* Kubernetes 1.13+ +* Kubernetes 1.13+ (CSI v1.0.0 compatibility) * Kubernetes has to allow privileged containers * Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`) ### 1. Create a secret with your S3 credentials -The endpoint is optional if you are using something else than AWS S3. Also the region can be empty if you are using some other S3 compatible storage. - ```yaml apiVersion: v1 kind: Secret @@ -30,11 +28,10 @@ stringData: endpoint: # If not on S3, set it to "" region: - # Currently only for s3ql - # If not using s3ql, set it to "" - encryptionKey: ``` +The region can be empty if you are using some other S3 compatible storage. + ### 2. Deploy the driver ```bash @@ -63,7 +60,7 @@ kubectl create -f pvc.yaml ```bash $ kubectl get pvc csi-s3-pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE -csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWX csi-s3 9s +csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWO csi-s3 9s ``` * Create a test pod which mounts your volume: @@ -96,7 +93,6 @@ The driver can be configured to use one of these mounters to mount buckets: * [rclone](https://rclone.org/commands/rclone_mount) * [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) * [goofys](https://github.com/kahing/goofys) -* [s3ql](https://github.com/s3ql/s3ql) * [s3backer](https://github.com/archiecobbs/s3backer) The mounter can be set as a parameter in the storage class. You can also create multiple storage classes for each mounter if you like. @@ -121,16 +117,7 @@ All mounters have different strengths and weaknesses depending on your use case. * Files can be viewed normally with any S3 client * Does not support appends or random writes -#### s3ql (not recommended*) - -* (Almost) full POSIX compatibility -* Uses its own object format -* Files are not readable with other S3 clients -* Support appends -* Supports compression before upload -* Supports encryption before upload - -#### s3backer (not recommended*) +#### s3backer (experimental*) * Represents a block device stored on S3 * Allows to use a real filesystem @@ -139,7 +126,8 @@ All mounters have different strengths and weaknesses depending on your use case. * Supports compression before upload (Not yet implemented in this driver) * Supports encryption before upload (Not yet implemented in this driver) -*s3ql and s3backer are not recommended at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod. +*s3backer is experimental at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod. +The s3backer binary is not bundled with the normal docker image to keep that as small as possible. Use the `-full` image tag for testing s3backer. Fore more detailed limitations consult the documentation of the different projects. diff --git a/cmd/s3driver/Dockerfile b/cmd/s3driver/Dockerfile index 3579a68..8e36c1a 100644 --- a/cmd/s3driver/Dockerfile +++ b/cmd/s3driver/Dockerfile @@ -2,15 +2,16 @@ FROM debian:stretch LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 slim image" +# s3fs and some other dependencies RUN apt-get update && \ apt-get install -y \ - s3fs wget unzip && \ + s3fs curl unzip && \ rm -rf /var/lib/apt/lists/* # install rclone -ARG RCLONE_VERSION=v1.46 +ARG RCLONE_VERSION=v1.47.0 RUN cd /tmp \ - && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ && rm -r /tmp/rclone* diff --git a/cmd/s3driver/Dockerfile.full b/cmd/s3driver/Dockerfile.full index ef266c5..58bfcd3 100644 --- a/cmd/s3driver/Dockerfile.full +++ b/cmd/s3driver/Dockerfile.full @@ -1,16 +1,3 @@ -FROM python:3.6 as s3ql-deps -LABEL maintainers="Cyrill Troxler " -LABEL description="s3ql dependencies" - -RUN apt-get update && \ - apt-get install -y \ - python3 python3-setuptools \ - python3-dev python3-pip pkg-config cython \ - libfuse-dev libattr1-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto - FROM debian:stretch as s3backer ARG S3BACKER_VERSION=1.5.0 @@ -33,36 +20,29 @@ WORKDIR /src/s3backer RUN git checkout tags/${S3BACKER_VERSION} RUN ./autogen.sh && \ - ./configure && \ - make && \ - make install + ./configure && \ + make && \ + make install -FROM python:3.6-slim +FROM debian:stretch LABEL maintainers="Cyrill Troxler " -LABEL description="csi-s3 full image" - -RUN apt-get update && \ - apt-get install -y \ - libfuse2 gcc sqlite3 libsqlite3-dev \ - s3fs psmisc procps libcurl3 xfsprogs wget unzip && \ - rm -rf /var/lib/apt/lists/* - -ARG S3QL_VERSION=2.29 -ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2 - -COPY --from=s3ql-deps /root/.cache /root/.cache -COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages -RUN pip install ${S3QL_URL} && rm -rf /root/.cache - +LABEL description="csi-s3 image" COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer +# s3fs and some other dependencies +RUN apt-get update && \ + apt-get install -y \ + libfuse2 gcc sqlite3 libsqlite3-dev \ + s3fs psmisc procps libcurl3 xfsprogs curl unzip && \ + rm -rf /var/lib/apt/lists/* + # install rclone -ARG RCLONE_VERSION=v1.46 +ARG RCLONE_VERSION=v1.47.0 RUN cd /tmp \ - && wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ - && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ - && rm -r /tmp/rclone* + && curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \ + && mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \ + && rm -r /tmp/rclone* COPY ./_output/s3driver /s3driver ENTRYPOINT ["/s3driver"] diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 033a872..52d03f7 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -9,6 +9,3 @@ stringData: endpoint: https://s3.eu-central-1.amazonaws.com # If not on S3, set it to "" region: - # Currently only for s3ql - # If not using s3ql, set it to "" - encryptionKey: "" diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index eca9192..1bd02ed 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -6,7 +6,7 @@ metadata: provisioner: ch.ctrox.csi.s3-driver parameters: # specify which mounter to use - # can be set to s3backer, s3ql, s3fs or goofys + # can be set to rclone, s3fs, goofys or s3backer mounter: rclone csi.storage.k8s.io/provisioner-secret-name: csi-s3-secret csi.storage.k8s.io/provisioner-secret-namespace: kube-system diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index 71fd98e..0ac2bf9 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -20,7 +20,6 @@ type Mounter interface { const ( s3fsMounterType = "s3fs" goofysMounterType = "goofys" - s3qlMounterType = "s3ql" s3backerMounterType = "s3backer" rcloneMounterType = "rclone" mounterTypeKey = "mounter" @@ -40,9 +39,6 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) { case goofysMounterType: return newGoofysMounter(bucket, cfg) - case s3qlMounterType: - return newS3qlMounter(bucket, cfg) - case s3backerMounterType: return newS3backerMounter(bucket, cfg) diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go deleted file mode 100644 index d32c47d..0000000 --- a/pkg/s3/mounter_s3ql.go +++ /dev/null @@ -1,114 +0,0 @@ -package s3 - -import ( - "bytes" - "fmt" - "net/url" - "os" - "os/exec" - "path" - "strings" - - "gopkg.in/ini.v1" -) - -// Implements Mounter -type s3qlMounter struct { - bucket *bucket - url string - bucketURL string - login string - password string - passphrase string - options []string - ssl bool - targetPath string -} - -const ( - s3qlCmdMkfs = "mkfs.s3ql" - s3qlCmdMount = "mount.s3ql" - s3qlCmdUnmount = "umount.s3ql" -) - -func newS3qlMounter(b *bucket, cfg *Config) (Mounter, error) { - url, err := url.Parse(cfg.Endpoint) - if err != nil { - return nil, err - } - ssl := url.Scheme != "http" - if strings.Contains(url.Scheme, "http") { - url.Scheme = "s3c" - } - s3ql := &s3qlMounter{ - bucket: b, - url: url.String(), - login: cfg.AccessKeyID, - password: cfg.SecretAccessKey, - passphrase: cfg.EncryptionKey, - ssl: ssl, - } - - // s3ql requires a trailing slash or it will just - // prepend the fspath to the s3ql files - url.Path = path.Join(url.Path, b.Name, b.FSPath) + "/" - s3ql.bucketURL = url.String() - - if !ssl { - s3ql.options = []string{"--backend-options", "no-ssl"} - } - - return s3ql, s3ql.writeConfig() -} - -func (s3ql *s3qlMounter) Stage(stagePath string) error { - // force creation to ignore existing data - args := []string{ - s3ql.bucketURL, - "--force", - } - - p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase) - reader := bytes.NewReader([]byte(p)) - cmd := exec.Command(s3qlCmdMkfs, append(args, s3ql.options...)...) - cmd.Stdin = reader - - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("Error running s3ql command: %s", out) - } - return nil -} - -func (s3ql *s3qlMounter) Unstage(stagePath string) error { - return nil -} - -func (s3ql *s3qlMounter) Mount(source string, target string) error { - args := []string{ - s3ql.bucketURL, - target, - "--allow-other", - } - return fuseMount(target, s3qlCmdMount, append(args, s3ql.options...)) -} - -func (s3ql *s3qlMounter) writeConfig() error { - s3qlIni := ini.Empty() - section, err := s3qlIni.NewSection("s3ql") - if err != nil { - return err - } - - section.NewKey("storage-url", s3ql.url) - section.NewKey("backend-login", s3ql.login) - section.NewKey("backend-password", s3ql.password) - section.NewKey("fs-passphrase", s3ql.passphrase) - - authDir := os.Getenv("HOME") + "/.s3ql" - authFile := authDir + "/authinfo2" - os.Mkdir(authDir, 0700) - s3qlIni.SaveTo(authFile) - os.Chmod(authFile, 0600) - return nil -} diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go index dd98fe5..c216e9b 100644 --- a/pkg/s3/s3-driver_suite_test.go +++ b/pkg/s3/s3-driver_suite_test.go @@ -65,33 +65,6 @@ var _ = Describe("S3Driver", func() { }) }) - Context("s3ql", func() { - socket := "/tmp/csi-s3ql.sock" - csiEndpoint := "unix://" + socket - - if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { - Expect(err).NotTo(HaveOccurred()) - } - driver, err := s3.NewS3("test-node", csiEndpoint) - if err != nil { - log.Fatal(err) - } - go driver.Run() - - Describe("CSI sanity", func() { - sanityCfg := &sanity.Config{ - TargetPath: os.TempDir() + "/s3ql-target", - StagingPath: os.TempDir() + "/s3ql-staging", - Address: csiEndpoint, - SecretsFile: "../../test/secret.yaml", - TestVolumeParameters: map[string]string{ - "mounter": "s3ql", - }, - } - sanity.GinkgoTest(sanityCfg) - }) - }) - Context("s3backer", func() { socket := "/tmp/csi-s3backer.sock" csiEndpoint := "unix://" + socket diff --git a/test/Dockerfile b/test/Dockerfile index b099b7f..3cb4237 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,4 +1,4 @@ -FROM ctrox/csi-s3:dev +FROM ctrox/csi-s3:dev-full LABEL maintainers="Cyrill Troxler " LABEL description="csi-s3 testing image"