Remove s3ql
s3ql does not work very well in a dynamic environment like k8s. Also as it needs a ton of dependencies just to get it built makes it hard to maintain.
This commit is contained in:
Cyrill Troxler
parent
76fc704c31
commit
f80104f215
10 changed files with 35 additions and 211 deletions
7
Makefile
7
Makefile
|
|
@ -16,8 +16,9 @@
|
|||
PROJECT_DIR=/app
|
||||
REGISTRY_NAME=ctrox
|
||||
IMAGE_NAME=csi-s3
|
||||
IMAGE_VERSION=1.0.1-alpha
|
||||
IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
|
||||
VERSION ?= dev
|
||||
IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(VERSION)
|
||||
FULL_IMAGE_TAG=$(IMAGE_TAG)-full
|
||||
TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test
|
||||
|
||||
build:
|
||||
|
|
@ -27,8 +28,10 @@ test:
|
|||
docker run --rm --privileged -v $(PWD):$(PROJECT_DIR) --device /dev/fuse $(TEST_IMAGE_TAG)
|
||||
container: build
|
||||
docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile .
|
||||
docker build -t $(FULL_IMAGE_TAG) --build-arg VERSION=$(VERSION) -f cmd/s3driver/Dockerfile.full .
|
||||
push: container
|
||||
docker push $(IMAGE_TAG)
|
||||
docker push $(FULL_IMAGE_TAG)
|
||||
clean:
|
||||
go clean -r -x
|
||||
-rm -rf _output
|
||||
|
|
|
|||
26
README.md
26
README.md
|
|
@ -10,14 +10,12 @@ This is still very experimental and should not be used in any production environ
|
|||
|
||||
### Requirements
|
||||
|
||||
* Kubernetes 1.13+
|
||||
* Kubernetes 1.13+ (CSI v1.0.0 compatibility)
|
||||
* Kubernetes has to allow privileged containers
|
||||
* Docker daemon must allow shared mounts (systemd flag `MountFlags=shared`)
|
||||
|
||||
### 1. Create a secret with your S3 credentials
|
||||
|
||||
The endpoint is optional if you are using something else than AWS S3. Also the region can be empty if you are using some other S3 compatible storage.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
|
|
@ -30,11 +28,10 @@ stringData:
|
|||
endpoint: <S3_ENDPOINT_URL>
|
||||
# If not on S3, set it to ""
|
||||
region: <S3_REGION>
|
||||
# Currently only for s3ql
|
||||
# If not using s3ql, set it to ""
|
||||
encryptionKey: <FS_ENCRYPTION_KEY>
|
||||
```
|
||||
|
||||
The region can be empty if you are using some other S3 compatible storage.
|
||||
|
||||
### 2. Deploy the driver
|
||||
|
||||
```bash
|
||||
|
|
@ -63,7 +60,7 @@ kubectl create -f pvc.yaml
|
|||
```bash
|
||||
$ kubectl get pvc csi-s3-pvc
|
||||
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
|
||||
csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWX csi-s3 9s
|
||||
csi-s3-pvc Bound pvc-c5d4634f-8507-11e8-9f33-0e243832354b 5Gi RWO csi-s3 9s
|
||||
```
|
||||
|
||||
* Create a test pod which mounts your volume:
|
||||
|
|
@ -96,7 +93,6 @@ The driver can be configured to use one of these mounters to mount buckets:
|
|||
* [rclone](https://rclone.org/commands/rclone_mount)
|
||||
* [s3fs](https://github.com/s3fs-fuse/s3fs-fuse)
|
||||
* [goofys](https://github.com/kahing/goofys)
|
||||
* [s3ql](https://github.com/s3ql/s3ql)
|
||||
* [s3backer](https://github.com/archiecobbs/s3backer)
|
||||
|
||||
The mounter can be set as a parameter in the storage class. You can also create multiple storage classes for each mounter if you like.
|
||||
|
|
@ -121,16 +117,7 @@ All mounters have different strengths and weaknesses depending on your use case.
|
|||
* Files can be viewed normally with any S3 client
|
||||
* Does not support appends or random writes
|
||||
|
||||
#### s3ql (not recommended*)
|
||||
|
||||
* (Almost) full POSIX compatibility
|
||||
* Uses its own object format
|
||||
* Files are not readable with other S3 clients
|
||||
* Support appends
|
||||
* Supports compression before upload
|
||||
* Supports encryption before upload
|
||||
|
||||
#### s3backer (not recommended*)
|
||||
#### s3backer (experimental*)
|
||||
|
||||
* Represents a block device stored on S3
|
||||
* Allows to use a real filesystem
|
||||
|
|
@ -139,7 +126,8 @@ All mounters have different strengths and weaknesses depending on your use case.
|
|||
* Supports compression before upload (Not yet implemented in this driver)
|
||||
* Supports encryption before upload (Not yet implemented in this driver)
|
||||
|
||||
*s3ql and s3backer are not recommended at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod.
|
||||
*s3backer is experimental at this point because volume corruption can occur pretty quickly in case of an unexpected shutdown of a Kubernetes node or CSI pod.
|
||||
The s3backer binary is not bundled with the normal docker image to keep that as small as possible. Use the `<version>-full` image tag for testing s3backer.
|
||||
|
||||
Fore more detailed limitations consult the documentation of the different projects.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,15 +2,16 @@ FROM debian:stretch
|
|||
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
|
||||
LABEL description="csi-s3 slim image"
|
||||
|
||||
# s3fs and some other dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
s3fs wget unzip && \
|
||||
s3fs curl unzip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# install rclone
|
||||
ARG RCLONE_VERSION=v1.46
|
||||
ARG RCLONE_VERSION=v1.47.0
|
||||
RUN cd /tmp \
|
||||
&& wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
|
||||
&& rm -r /tmp/rclone*
|
||||
|
|
|
|||
|
|
@ -1,16 +1,3 @@
|
|||
FROM python:3.6 as s3ql-deps
|
||||
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
|
||||
LABEL description="s3ql dependencies"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
python3 python3-setuptools \
|
||||
python3-dev python3-pip pkg-config cython \
|
||||
libfuse-dev libattr1-dev && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip3 install llfuse apsw defusedxml dugong requests pycrypto
|
||||
|
||||
FROM debian:stretch as s3backer
|
||||
ARG S3BACKER_VERSION=1.5.0
|
||||
|
||||
|
|
@ -33,36 +20,29 @@ WORKDIR /src/s3backer
|
|||
RUN git checkout tags/${S3BACKER_VERSION}
|
||||
|
||||
RUN ./autogen.sh && \
|
||||
./configure && \
|
||||
make && \
|
||||
make install
|
||||
./configure && \
|
||||
make && \
|
||||
make install
|
||||
|
||||
FROM python:3.6-slim
|
||||
FROM debian:stretch
|
||||
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
|
||||
LABEL description="csi-s3 full image"
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
libfuse2 gcc sqlite3 libsqlite3-dev \
|
||||
s3fs psmisc procps libcurl3 xfsprogs wget unzip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG S3QL_VERSION=2.29
|
||||
ENV S3QL_URL=https://github.com/s3ql/s3ql/releases/download/release-${S3QL_VERSION}/s3ql-${S3QL_VERSION}.tar.bz2
|
||||
|
||||
COPY --from=s3ql-deps /root/.cache /root/.cache
|
||||
COPY --from=s3ql-deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages
|
||||
RUN pip install ${S3QL_URL} && rm -rf /root/.cache
|
||||
|
||||
LABEL description="csi-s3 image"
|
||||
COPY --from=s3backer /usr/bin/s3backer /usr/bin/s3backer
|
||||
|
||||
# s3fs and some other dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
libfuse2 gcc sqlite3 libsqlite3-dev \
|
||||
s3fs psmisc procps libcurl3 xfsprogs curl unzip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# install rclone
|
||||
ARG RCLONE_VERSION=v1.46
|
||||
ARG RCLONE_VERSION=v1.47.0
|
||||
RUN cd /tmp \
|
||||
&& wget -q https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
|
||||
&& rm -r /tmp/rclone*
|
||||
&& curl -O https://downloads.rclone.org/${RCLONE_VERSION}/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& unzip /tmp/rclone-${RCLONE_VERSION}-linux-amd64.zip \
|
||||
&& mv /tmp/rclone-*-linux-amd64/rclone /usr/bin \
|
||||
&& rm -r /tmp/rclone*
|
||||
|
||||
COPY ./_output/s3driver /s3driver
|
||||
ENTRYPOINT ["/s3driver"]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,3 @@ stringData:
|
|||
endpoint: https://s3.eu-central-1.amazonaws.com
|
||||
# If not on S3, set it to ""
|
||||
region: <S3_REGION>
|
||||
# Currently only for s3ql
|
||||
# If not using s3ql, set it to ""
|
||||
encryptionKey: ""
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ metadata:
|
|||
provisioner: ch.ctrox.csi.s3-driver
|
||||
parameters:
|
||||
# specify which mounter to use
|
||||
# can be set to s3backer, s3ql, s3fs or goofys
|
||||
# can be set to rclone, s3fs, goofys or s3backer
|
||||
mounter: rclone
|
||||
csi.storage.k8s.io/provisioner-secret-name: csi-s3-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: kube-system
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ type Mounter interface {
|
|||
const (
|
||||
s3fsMounterType = "s3fs"
|
||||
goofysMounterType = "goofys"
|
||||
s3qlMounterType = "s3ql"
|
||||
s3backerMounterType = "s3backer"
|
||||
rcloneMounterType = "rclone"
|
||||
mounterTypeKey = "mounter"
|
||||
|
|
@ -40,9 +39,6 @@ func newMounter(bucket *bucket, cfg *Config) (Mounter, error) {
|
|||
case goofysMounterType:
|
||||
return newGoofysMounter(bucket, cfg)
|
||||
|
||||
case s3qlMounterType:
|
||||
return newS3qlMounter(bucket, cfg)
|
||||
|
||||
case s3backerMounterType:
|
||||
return newS3backerMounter(bucket, cfg)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,114 +0,0 @@
|
|||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/ini.v1"
|
||||
)
|
||||
|
||||
// Implements Mounter
|
||||
type s3qlMounter struct {
|
||||
bucket *bucket
|
||||
url string
|
||||
bucketURL string
|
||||
login string
|
||||
password string
|
||||
passphrase string
|
||||
options []string
|
||||
ssl bool
|
||||
targetPath string
|
||||
}
|
||||
|
||||
const (
|
||||
s3qlCmdMkfs = "mkfs.s3ql"
|
||||
s3qlCmdMount = "mount.s3ql"
|
||||
s3qlCmdUnmount = "umount.s3ql"
|
||||
)
|
||||
|
||||
func newS3qlMounter(b *bucket, cfg *Config) (Mounter, error) {
|
||||
url, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ssl := url.Scheme != "http"
|
||||
if strings.Contains(url.Scheme, "http") {
|
||||
url.Scheme = "s3c"
|
||||
}
|
||||
s3ql := &s3qlMounter{
|
||||
bucket: b,
|
||||
url: url.String(),
|
||||
login: cfg.AccessKeyID,
|
||||
password: cfg.SecretAccessKey,
|
||||
passphrase: cfg.EncryptionKey,
|
||||
ssl: ssl,
|
||||
}
|
||||
|
||||
// s3ql requires a trailing slash or it will just
|
||||
// prepend the fspath to the s3ql files
|
||||
url.Path = path.Join(url.Path, b.Name, b.FSPath) + "/"
|
||||
s3ql.bucketURL = url.String()
|
||||
|
||||
if !ssl {
|
||||
s3ql.options = []string{"--backend-options", "no-ssl"}
|
||||
}
|
||||
|
||||
return s3ql, s3ql.writeConfig()
|
||||
}
|
||||
|
||||
func (s3ql *s3qlMounter) Stage(stagePath string) error {
|
||||
// force creation to ignore existing data
|
||||
args := []string{
|
||||
s3ql.bucketURL,
|
||||
"--force",
|
||||
}
|
||||
|
||||
p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase)
|
||||
reader := bytes.NewReader([]byte(p))
|
||||
cmd := exec.Command(s3qlCmdMkfs, append(args, s3ql.options...)...)
|
||||
cmd.Stdin = reader
|
||||
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error running s3ql command: %s", out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s3ql *s3qlMounter) Unstage(stagePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s3ql *s3qlMounter) Mount(source string, target string) error {
|
||||
args := []string{
|
||||
s3ql.bucketURL,
|
||||
target,
|
||||
"--allow-other",
|
||||
}
|
||||
return fuseMount(target, s3qlCmdMount, append(args, s3ql.options...))
|
||||
}
|
||||
|
||||
func (s3ql *s3qlMounter) writeConfig() error {
|
||||
s3qlIni := ini.Empty()
|
||||
section, err := s3qlIni.NewSection("s3ql")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
section.NewKey("storage-url", s3ql.url)
|
||||
section.NewKey("backend-login", s3ql.login)
|
||||
section.NewKey("backend-password", s3ql.password)
|
||||
section.NewKey("fs-passphrase", s3ql.passphrase)
|
||||
|
||||
authDir := os.Getenv("HOME") + "/.s3ql"
|
||||
authFile := authDir + "/authinfo2"
|
||||
os.Mkdir(authDir, 0700)
|
||||
s3qlIni.SaveTo(authFile)
|
||||
os.Chmod(authFile, 0600)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -65,33 +65,6 @@ var _ = Describe("S3Driver", func() {
|
|||
})
|
||||
})
|
||||
|
||||
Context("s3ql", func() {
|
||||
socket := "/tmp/csi-s3ql.sock"
|
||||
csiEndpoint := "unix://" + socket
|
||||
|
||||
if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
driver, err := s3.NewS3("test-node", csiEndpoint)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
go driver.Run()
|
||||
|
||||
Describe("CSI sanity", func() {
|
||||
sanityCfg := &sanity.Config{
|
||||
TargetPath: os.TempDir() + "/s3ql-target",
|
||||
StagingPath: os.TempDir() + "/s3ql-staging",
|
||||
Address: csiEndpoint,
|
||||
SecretsFile: "../../test/secret.yaml",
|
||||
TestVolumeParameters: map[string]string{
|
||||
"mounter": "s3ql",
|
||||
},
|
||||
}
|
||||
sanity.GinkgoTest(sanityCfg)
|
||||
})
|
||||
})
|
||||
|
||||
Context("s3backer", func() {
|
||||
socket := "/tmp/csi-s3backer.sock"
|
||||
csiEndpoint := "unix://" + socket
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
FROM ctrox/csi-s3:dev
|
||||
FROM ctrox/csi-s3:dev-full
|
||||
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
|
||||
LABEL description="csi-s3 testing image"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue