From 13eba47da6136cd8bcb3be81b70a3b87cea8af9b Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Mon, 16 Jul 2018 22:27:45 +0200 Subject: [PATCH] Add experimental s3ql mounter --- Gopkg.lock | 8 +- Gopkg.toml | 4 + Makefile | 4 +- cmd/s3driver/Dockerfile.s3ql | 23 +++++ cmd/s3driver/main.go | 2 + deploy/kubernetes/csi-s3-driver.yaml | 8 +- deploy/kubernetes/provisioner.yaml | 8 +- deploy/kubernetes/pvc.yaml | 2 +- deploy/kubernetes/secret.yaml | 2 + deploy/kubernetes/storageclass.yaml | 4 +- pkg/s3/config.go | 2 + pkg/s3/controllerserver.go | 18 +++- pkg/s3/nodeserver.go | 53 ++++++----- pkg/s3/s3-client.go | 4 - pkg/s3/s3-driver_test.go | 1 + pkg/s3/s3ql.go | 129 +++++++++++++++++++++++++++ test/Dockerfile | 18 +++- 17 files changed, 247 insertions(+), 43 deletions(-) create mode 100644 cmd/s3driver/Dockerfile.s3ql create mode 100644 pkg/s3/s3ql.go diff --git a/Gopkg.lock b/Gopkg.lock index 4002686..40ababf 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -334,6 +334,12 @@ revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" version = "v1.13.0" +[[projects]] + name = "gopkg.in/ini.v1" + packages = ["."] + revision = "358ee7663966325963d4e8b2e1fbd570c5195153" + version = "v1.38.1" + [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] @@ -366,6 +372,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4" + inputs-digest = "9bd4175acb8ce47fe57c3d859bc6b061a6c5dd3017f57777f3fefb27d0020d75" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1a7f0e9..eaa6645 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,3 +37,7 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "gopkg.in/ini.v1" + version = "1.38.1" diff --git a/Makefile b/Makefile index 7c9b476..1b55be7 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver REGISTRY_NAME=ctrox IMAGE_NAME=csi-s3-driver -IMAGE_VERSION=0.1.0 +IMAGE_VERSION=0.2.0 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test @@ -27,7 +27,7 @@ test: docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile . docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG) container: build - docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile . + docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql . push: container docker push $(IMAGE_TAG) clean: diff --git a/cmd/s3driver/Dockerfile.s3ql b/cmd/s3driver/Dockerfile.s3ql new file mode 100644 index 0000000..d73f638 --- /dev/null +++ b/cmd/s3driver/Dockerfile.s3ql @@ -0,0 +1,23 @@ +FROM debian:stretch +LABEL maintainers="Cyrill Troxler " +LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install + +COPY ./_output/s3driver /s3driver +ENTRYPOINT ["/s3driver"] diff --git a/cmd/s3driver/main.go b/cmd/s3driver/main.go index 1d9eee7..f071b10 100644 --- a/cmd/s3driver/main.go +++ b/cmd/s3driver/main.go @@ -35,6 +35,7 @@ var ( secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use") s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use") region = flag.String("region", "", "S3 Region to use") + encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)") ) func main() { @@ -45,6 +46,7 @@ func main() { SecretAccessKey: *secretAccessKey, Endpoint: *s3endpoint, Region: *region, + EncryptionKey: *encryptionKey, } driver, err := s3.NewS3(*nodeID, *endpoint, cfg) diff --git a/deploy/kubernetes/csi-s3-driver.yaml b/deploy/kubernetes/csi-s3-driver.yaml index cc8443a..931a831 100644 --- a/deploy/kubernetes/csi-s3-driver.yaml +++ b/deploy/kubernetes/csi-s3-driver.yaml @@ -75,7 +75,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -83,6 +83,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -111,6 +112,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: plugin-dir diff --git a/deploy/kubernetes/provisioner.yaml b/deploy/kubernetes/provisioner.yaml index 1be1b5d..11f9e97 100644 --- a/deploy/kubernetes/provisioner.yaml +++ b/deploy/kubernetes/provisioner.yaml @@ -79,7 +79,7 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver - name: s3-csi-driver - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -87,6 +87,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -115,6 +116,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: socket-dir diff --git a/deploy/kubernetes/pvc.yaml b/deploy/kubernetes/pvc.yaml index a645b4e..68f0496 100644 --- a/deploy/kubernetes/pvc.yaml +++ b/deploy/kubernetes/pvc.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 266157c..b0119fa 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -8,3 +8,5 @@ stringData: endpoint: # If not on S3, just set it to "" region: + # Currently only for s3ql + # encryptionKey: diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index 3db2aef..a6dfd9b 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -6,6 +6,6 @@ metadata: provisioner: ch.ctrox.csi.s3-driver parameters: # specify which mounter to use - # can be set to s3fs or goofys + # can be set to s3fs, goofys or s3ql # s3fs is the default - # mounter: s3fs + # mounter: s3ql diff --git a/pkg/s3/config.go b/pkg/s3/config.go index 43eecc0..f5b1714 100644 --- a/pkg/s3/config.go +++ b/pkg/s3/config.go @@ -6,4 +6,6 @@ type Config struct { SecretAccessKey string Region string Endpoint string + Mounter string + EncryptionKey string } diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 9f2c4f7..6bfca9b 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -60,6 +60,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } } + mounter := cs.s3.cfg.Mounter + if mounter == "" { + mounter = req.GetParameters()[mounterKey] + } + switch mounter { + case s3qlMounter: + if err := s3qlCreate(volumeID, cs.s3.cfg); err != nil { + return nil, err + } + } glog.V(4).Infof("create volume %s", volumeID) s3Vol := s3Volume{} @@ -123,6 +133,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId())) } - // We currently support all capabilities - return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil + for _, cap := range req.VolumeCapabilities { + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil + } + } + return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil } diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 5b7f28f..9fd88f5 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -35,6 +35,7 @@ const ( mounterKey = "mounter" s3fsMounter = "s3fs" goofysMounter = "goofys" + s3qlMounter = "s3ql" ) type nodeServer struct { @@ -85,17 +86,26 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - mounter, exists := attrib[mounterKey] - if !exists || mounter == s3fsMounter { + mounter := ns.s3.cfg.Mounter + if mounter == "" { + mounter = attrib[mounterKey] + } + switch mounter { + case "": + case s3fsMounter: if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil { return nil, err } - } else if mounter == goofysMounter { + case goofysMounter: if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil { return nil, err } - } else { - return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter) + case s3qlMounter: + if err := s3qlMount(volumeID, ns.s3.cfg, targetPath); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, ns.s3.cfg.Mounter) } glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath) @@ -121,29 +131,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return &csi.NodeUnpublishVolumeResponse{}, nil } - -func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeStageVolumeResponse{}, nil +func (ns *nodeServer) NodeStageVolume( + ctx context.Context, + req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } -func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeUnstageVolumeResponse{}, nil +func (ns *nodeServer) NodeUnstageVolume( + ctx context.Context, + req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/s3/s3-client.go b/pkg/s3/s3-client.go index 7fff9a4..76713a3 100644 --- a/pkg/s3/s3-client.go +++ b/pkg/s3/s3-client.go @@ -17,10 +17,6 @@ type s3Client struct { minio *minio.Client } -type bucketMetadata struct { - CapacityBytes int64 -} - func newS3Client(cfg *Config) (*s3Client, error) { var client = &s3Client{} diff --git a/pkg/s3/s3-driver_test.go b/pkg/s3/s3-driver_test.go index 4a8e875..89b84c4 100644 --- a/pkg/s3/s3-driver_test.go +++ b/pkg/s3/s3-driver_test.go @@ -36,6 +36,7 @@ func TestDriver(t *testing.T) { AccessKeyID: "FJDSJ", SecretAccessKey: "DSG643HGDS", Endpoint: "http://127.0.0.1:9000", + EncryptionKey: "IskEwCuEg6drywi", } driver, err := NewS3("test-node", endpoint, cfg) if err != nil { diff --git a/pkg/s3/s3ql.go b/pkg/s3/s3ql.go new file mode 100644 index 0000000..3e1a7b8 --- /dev/null +++ b/pkg/s3/s3ql.go @@ -0,0 +1,129 @@ +package s3 + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "strings" + + "gopkg.in/ini.v1" +) + +type s3fsConfig struct { + url string + bucketURL string + login string + password string + passphrase string + options []string + ssl bool + targetPath string +} + +const ( + s3qlCmdMkfs = "mkfs.s3ql" + s3qlCmdMount = "mount.s3ql" +) + +func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) { + url, err := url.Parse(cfg.Endpoint) + if err != nil { + return nil, err + } + ssl := url.Scheme != "http" + if strings.Contains(url.Scheme, "http") { + url.Scheme = "s3c" + } + s3ql := &s3fsConfig{ + url: url.String(), + login: cfg.AccessKeyID, + password: cfg.SecretAccessKey, + passphrase: cfg.EncryptionKey, + ssl: ssl, + targetPath: targetPath, + } + + url.Path = path.Join(url.Path, bucket) + s3ql.bucketURL = url.String() + + if !ssl { + s3ql.options = []string{"--backend-options", "no-ssl"} + } + + return s3ql, s3ql.writeConfig() +} + +func s3qlCreate(bucket string, cfg *Config) error { + s3ql, err := newS3ql(bucket, "unknown", cfg) + if err != nil { + return err + } + return s3ql.create() +} + +func s3qlMount(bucket string, cfg *Config, targetPath string) error { + s3ql, err := newS3ql(bucket, targetPath, cfg) + if err != nil { + return err + } + + return s3ql.mount() +} + +func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { + cmd := exec.Command(s3qlCmd, args...) + if stdin != nil { + cmd.Stdin = stdin + } + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Error running s3ql command: %s", out) + } + return nil +} + +func (cfg *s3fsConfig) create() error { + // force creation to ignore existing data + args := []string{ + cfg.bucketURL, + "--force", + } + + p := fmt.Sprintf("%s\n%s\n", cfg.passphrase, cfg.passphrase) + reader := bytes.NewReader([]byte(p)) + return s3qlCmd(s3qlCmdMkfs, append(args, cfg.options...), reader) +} + +func (cfg *s3fsConfig) mount() error { + args := []string{ + cfg.bucketURL, + cfg.targetPath, + "--allow-other", + } + return s3qlCmd(s3qlCmdMount, append(args, cfg.options...), nil) +} + +func (cfg *s3fsConfig) writeConfig() error { + s3qlIni := ini.Empty() + section, err := s3qlIni.NewSection("s3ql") + if err != nil { + return err + } + + section.NewKey("storage-url", cfg.url) + section.NewKey("backend-login", cfg.login) + section.NewKey("backend-password", cfg.password) + section.NewKey("fs-passphrase", cfg.passphrase) + + authDir := os.Getenv("HOME") + "/.s3ql" + authFile := authDir + "/authinfo2" + os.Mkdir(authDir, 0700) + s3qlIni.SaveTo(authFile) + os.Chmod(authFile, 0600) + return nil +} diff --git a/test/Dockerfile b/test/Dockerfile index f586156..3f0a12b 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,8 +1,24 @@ FROM golang:stretch LABEL maintainers="Cyrill Troxler " LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install -RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/* RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"]