From 13eba47da6136cd8bcb3be81b70a3b87cea8af9b Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Mon, 16 Jul 2018 22:27:45 +0200 Subject: [PATCH 1/8] Add experimental s3ql mounter --- Gopkg.lock | 8 +- Gopkg.toml | 4 + Makefile | 4 +- cmd/s3driver/Dockerfile.s3ql | 23 +++++ cmd/s3driver/main.go | 2 + deploy/kubernetes/csi-s3-driver.yaml | 8 +- deploy/kubernetes/provisioner.yaml | 8 +- deploy/kubernetes/pvc.yaml | 2 +- deploy/kubernetes/secret.yaml | 2 + deploy/kubernetes/storageclass.yaml | 4 +- pkg/s3/config.go | 2 + pkg/s3/controllerserver.go | 18 +++- pkg/s3/nodeserver.go | 53 ++++++----- pkg/s3/s3-client.go | 4 - pkg/s3/s3-driver_test.go | 1 + pkg/s3/s3ql.go | 129 +++++++++++++++++++++++++++ test/Dockerfile | 18 +++- 17 files changed, 247 insertions(+), 43 deletions(-) create mode 100644 cmd/s3driver/Dockerfile.s3ql create mode 100644 pkg/s3/s3ql.go diff --git a/Gopkg.lock b/Gopkg.lock index 4002686..40ababf 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -334,6 +334,12 @@ revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" version = "v1.13.0" +[[projects]] + name = "gopkg.in/ini.v1" + packages = ["."] + revision = "358ee7663966325963d4e8b2e1fbd570c5195153" + version = "v1.38.1" + [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] @@ -366,6 +372,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4" + inputs-digest = "9bd4175acb8ce47fe57c3d859bc6b061a6c5dd3017f57777f3fefb27d0020d75" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1a7f0e9..eaa6645 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -37,3 +37,7 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "gopkg.in/ini.v1" + version = "1.38.1" diff --git a/Makefile b/Makefile index 7c9b476..1b55be7 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver REGISTRY_NAME=ctrox IMAGE_NAME=csi-s3-driver -IMAGE_VERSION=0.1.0 +IMAGE_VERSION=0.2.0 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test @@ -27,7 +27,7 @@ test: docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile . docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG) container: build - docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile . + docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql . push: container docker push $(IMAGE_TAG) clean: diff --git a/cmd/s3driver/Dockerfile.s3ql b/cmd/s3driver/Dockerfile.s3ql new file mode 100644 index 0000000..d73f638 --- /dev/null +++ b/cmd/s3driver/Dockerfile.s3ql @@ -0,0 +1,23 @@ +FROM debian:stretch +LABEL maintainers="Cyrill Troxler " +LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install + +COPY ./_output/s3driver /s3driver +ENTRYPOINT ["/s3driver"] diff --git a/cmd/s3driver/main.go b/cmd/s3driver/main.go index 1d9eee7..f071b10 100644 --- a/cmd/s3driver/main.go +++ b/cmd/s3driver/main.go @@ -35,6 +35,7 @@ var ( secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use") s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use") region = flag.String("region", "", "S3 Region to use") + encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)") ) func main() { @@ -45,6 +46,7 @@ func main() { SecretAccessKey: *secretAccessKey, Endpoint: *s3endpoint, Region: *region, + EncryptionKey: *encryptionKey, } driver, err := s3.NewS3(*nodeID, *endpoint, cfg) diff --git a/deploy/kubernetes/csi-s3-driver.yaml b/deploy/kubernetes/csi-s3-driver.yaml index cc8443a..931a831 100644 --- a/deploy/kubernetes/csi-s3-driver.yaml +++ b/deploy/kubernetes/csi-s3-driver.yaml @@ -75,7 +75,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -83,6 +83,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -111,6 +112,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: plugin-dir diff --git a/deploy/kubernetes/provisioner.yaml b/deploy/kubernetes/provisioner.yaml index 1be1b5d..11f9e97 100644 --- a/deploy/kubernetes/provisioner.yaml +++ b/deploy/kubernetes/provisioner.yaml @@ -79,7 +79,7 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver - name: s3-csi-driver - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -87,6 +87,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -115,6 +116,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: socket-dir diff --git a/deploy/kubernetes/pvc.yaml b/deploy/kubernetes/pvc.yaml index a645b4e..68f0496 100644 --- a/deploy/kubernetes/pvc.yaml +++ b/deploy/kubernetes/pvc.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 266157c..b0119fa 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -8,3 +8,5 @@ stringData: endpoint: # If not on S3, just set it to "" region: + # Currently only for s3ql + # encryptionKey: diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index 3db2aef..a6dfd9b 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -6,6 +6,6 @@ metadata: provisioner: ch.ctrox.csi.s3-driver parameters: # specify which mounter to use - # can be set to s3fs or goofys + # can be set to s3fs, goofys or s3ql # s3fs is the default - # mounter: s3fs + # mounter: s3ql diff --git a/pkg/s3/config.go b/pkg/s3/config.go index 43eecc0..f5b1714 100644 --- a/pkg/s3/config.go +++ b/pkg/s3/config.go @@ -6,4 +6,6 @@ type Config struct { SecretAccessKey string Region string Endpoint string + Mounter string + EncryptionKey string } diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 9f2c4f7..6bfca9b 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -60,6 +60,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } } + mounter := cs.s3.cfg.Mounter + if mounter == "" { + mounter = req.GetParameters()[mounterKey] + } + switch mounter { + case s3qlMounter: + if err := s3qlCreate(volumeID, cs.s3.cfg); err != nil { + return nil, err + } + } glog.V(4).Infof("create volume %s", volumeID) s3Vol := s3Volume{} @@ -123,6 +133,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId())) } - // We currently support all capabilities - return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil + for _, cap := range req.VolumeCapabilities { + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil + } + } + return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil } diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 5b7f28f..9fd88f5 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -35,6 +35,7 @@ const ( mounterKey = "mounter" s3fsMounter = "s3fs" goofysMounter = "goofys" + s3qlMounter = "s3ql" ) type nodeServer struct { @@ -85,17 +86,26 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - mounter, exists := attrib[mounterKey] - if !exists || mounter == s3fsMounter { + mounter := ns.s3.cfg.Mounter + if mounter == "" { + mounter = attrib[mounterKey] + } + switch mounter { + case "": + case s3fsMounter: if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil { return nil, err } - } else if mounter == goofysMounter { + case goofysMounter: if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil { return nil, err } - } else { - return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter) + case s3qlMounter: + if err := s3qlMount(volumeID, ns.s3.cfg, targetPath); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, ns.s3.cfg.Mounter) } glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath) @@ -121,29 +131,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return &csi.NodeUnpublishVolumeResponse{}, nil } - -func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeStageVolumeResponse{}, nil +func (ns *nodeServer) NodeStageVolume( + ctx context.Context, + req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } -func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeUnstageVolumeResponse{}, nil +func (ns *nodeServer) NodeUnstageVolume( + ctx context.Context, + req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/s3/s3-client.go b/pkg/s3/s3-client.go index 7fff9a4..76713a3 100644 --- a/pkg/s3/s3-client.go +++ b/pkg/s3/s3-client.go @@ -17,10 +17,6 @@ type s3Client struct { minio *minio.Client } -type bucketMetadata struct { - CapacityBytes int64 -} - func newS3Client(cfg *Config) (*s3Client, error) { var client = &s3Client{} diff --git a/pkg/s3/s3-driver_test.go b/pkg/s3/s3-driver_test.go index 4a8e875..89b84c4 100644 --- a/pkg/s3/s3-driver_test.go +++ b/pkg/s3/s3-driver_test.go @@ -36,6 +36,7 @@ func TestDriver(t *testing.T) { AccessKeyID: "FJDSJ", SecretAccessKey: "DSG643HGDS", Endpoint: "http://127.0.0.1:9000", + EncryptionKey: "IskEwCuEg6drywi", } driver, err := NewS3("test-node", endpoint, cfg) if err != nil { diff --git a/pkg/s3/s3ql.go b/pkg/s3/s3ql.go new file mode 100644 index 0000000..3e1a7b8 --- /dev/null +++ b/pkg/s3/s3ql.go @@ -0,0 +1,129 @@ +package s3 + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "strings" + + "gopkg.in/ini.v1" +) + +type s3fsConfig struct { + url string + bucketURL string + login string + password string + passphrase string + options []string + ssl bool + targetPath string +} + +const ( + s3qlCmdMkfs = "mkfs.s3ql" + s3qlCmdMount = "mount.s3ql" +) + +func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) { + url, err := url.Parse(cfg.Endpoint) + if err != nil { + return nil, err + } + ssl := url.Scheme != "http" + if strings.Contains(url.Scheme, "http") { + url.Scheme = "s3c" + } + s3ql := &s3fsConfig{ + url: url.String(), + login: cfg.AccessKeyID, + password: cfg.SecretAccessKey, + passphrase: cfg.EncryptionKey, + ssl: ssl, + targetPath: targetPath, + } + + url.Path = path.Join(url.Path, bucket) + s3ql.bucketURL = url.String() + + if !ssl { + s3ql.options = []string{"--backend-options", "no-ssl"} + } + + return s3ql, s3ql.writeConfig() +} + +func s3qlCreate(bucket string, cfg *Config) error { + s3ql, err := newS3ql(bucket, "unknown", cfg) + if err != nil { + return err + } + return s3ql.create() +} + +func s3qlMount(bucket string, cfg *Config, targetPath string) error { + s3ql, err := newS3ql(bucket, targetPath, cfg) + if err != nil { + return err + } + + return s3ql.mount() +} + +func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { + cmd := exec.Command(s3qlCmd, args...) + if stdin != nil { + cmd.Stdin = stdin + } + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Error running s3ql command: %s", out) + } + return nil +} + +func (cfg *s3fsConfig) create() error { + // force creation to ignore existing data + args := []string{ + cfg.bucketURL, + "--force", + } + + p := fmt.Sprintf("%s\n%s\n", cfg.passphrase, cfg.passphrase) + reader := bytes.NewReader([]byte(p)) + return s3qlCmd(s3qlCmdMkfs, append(args, cfg.options...), reader) +} + +func (cfg *s3fsConfig) mount() error { + args := []string{ + cfg.bucketURL, + cfg.targetPath, + "--allow-other", + } + return s3qlCmd(s3qlCmdMount, append(args, cfg.options...), nil) +} + +func (cfg *s3fsConfig) writeConfig() error { + s3qlIni := ini.Empty() + section, err := s3qlIni.NewSection("s3ql") + if err != nil { + return err + } + + section.NewKey("storage-url", cfg.url) + section.NewKey("backend-login", cfg.login) + section.NewKey("backend-password", cfg.password) + section.NewKey("fs-passphrase", cfg.passphrase) + + authDir := os.Getenv("HOME") + "/.s3ql" + authFile := authDir + "/authinfo2" + os.Mkdir(authDir, 0700) + s3qlIni.SaveTo(authFile) + os.Chmod(authFile, 0600) + return nil +} diff --git a/test/Dockerfile b/test/Dockerfile index f586156..3f0a12b 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,8 +1,24 @@ FROM golang:stretch LABEL maintainers="Cyrill Troxler " LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install -RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/* RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"] From 0bd6f9b7cc63376e025622b7b956b9ed19d75fd3 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Thu, 19 Jul 2018 19:17:53 +0200 Subject: [PATCH 2/8] Use new csi-test release --- Gopkg.lock | 6 +++--- Gopkg.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 40ababf..d56885f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -125,13 +125,13 @@ version = "v0.19.0" [[projects]] - branch = "master" name = "github.com/kubernetes-csi/csi-test" packages = [ "pkg/sanity", "utils" ] - revision = "6fed82d24d3a04c1814440d4178e7bf8ae9e67e6" + revision = "718c9544f5e16cba31881333e0f9dff371663dea" + version = "v0.3.0-1" [[projects]] branch = "master" @@ -372,6 +372,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9bd4175acb8ce47fe57c3d859bc6b061a6c5dd3017f57777f3fefb27d0020d75" + inputs-digest = "659f47734b56af7fba146039231841e82cc4c3c95dff2814ec3688e967790a50" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index eaa6645..98f9a1b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -11,8 +11,8 @@ version = "0.19.0" [[constraint]] - branch = "master" name = "github.com/kubernetes-csi/csi-test" + version = "v0.3.0-1" [[constraint]] branch = "master" From 093c5bf500b2ff5fb233deebe09864cf6d57a051 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Thu, 19 Jul 2018 22:04:36 +0200 Subject: [PATCH 3/8] Use ginkgo test suite to test all mounters --- pkg/s3/controllerserver.go | 2 +- pkg/s3/s3-driver_suite_test.go | 116 +++++++++++++++++++++++++++++++++ pkg/s3/s3-driver_test.go | 40 ++---------- 3 files changed, 122 insertions(+), 36 deletions(-) create mode 100644 pkg/s3/s3-driver_suite_test.go diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 6bfca9b..8b2f4bc 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -78,7 +78,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ Id: volumeID, - CapacityBytes: 0, + CapacityBytes: 1, Attributes: req.GetParameters(), }, }, nil diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go new file mode 100644 index 0000000..47b3ce6 --- /dev/null +++ b/pkg/s3/s3-driver_suite_test.go @@ -0,0 +1,116 @@ +package s3_test + +import ( + "io/ioutil" + "log" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/ctrox/csi-s3-driver/pkg/s3" + "github.com/kubernetes-csi/csi-test/pkg/sanity" +) + +const () + +var _ = Describe("S3Driver", func() { + mntDir, err := ioutil.TempDir("", "mnt") + if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + + AfterSuite(func() { + os.RemoveAll(mntDir) + }) + + Context("goofys", func() { + socket := "/tmp/csi-goofys.sock" + csiEndpoint := "unix://" + socket + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "goofys", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + + Context("s3fs", func() { + socket := "/tmp/csi-s3fs.sock" + csiEndpoint := "unix://" + socket + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "s3fs", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + defer os.RemoveAll(mntDir) + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + + Context("s3ql", func() { + socket := "/tmp/csi-s3ql.sock" + csiEndpoint := "unix://" + socket + + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "s3ql", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + defer os.RemoveAll(mntDir) + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + +}) diff --git a/pkg/s3/s3-driver_test.go b/pkg/s3/s3-driver_test.go index 89b84c4..5588247 100644 --- a/pkg/s3/s3-driver_test.go +++ b/pkg/s3/s3-driver_test.go @@ -17,43 +17,13 @@ limitations under the License. package s3 import ( - "io/ioutil" - "log" - "os" "testing" - "github.com/kubernetes-csi/csi-test/pkg/sanity" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) -func TestDriver(t *testing.T) { - socket := "/tmp/csi.sock" - endpoint := "unix://" + socket - - if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to remove unix domain socket file %s, error: %s", socket, err) - } - cfg := &Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - EncryptionKey: "IskEwCuEg6drywi", - } - driver, err := NewS3("test-node", endpoint, cfg) - if err != nil { - log.Fatal(err) - } - go driver.Run() - - mntDir, err := ioutil.TempDir("", "mnt") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(mntDir) - - sanityCfg := &sanity.Config{ - TargetPath: mntDir, - Address: endpoint, - } - - sanity.Test(t, sanityCfg) +func TestS3Driver(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "S3Driver") } From 9d5d84ebfb83991cae38c5a93ae81d16eed95834 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 21 Jul 2018 13:35:31 +0200 Subject: [PATCH 4/8] Refactor all mounters to use the mounter interface --- pkg/s3/controllerserver.go | 19 +++++---- pkg/s3/goofys.go | 37 ---------------- pkg/s3/mounter.go | 34 +++++++++++++++ pkg/s3/mounter_goofys.go | 63 ++++++++++++++++++++++++++++ pkg/s3/mounter_s3fs.go | 64 ++++++++++++++++++++++++++++ pkg/s3/{s3ql.go => mounter_s3ql.go} | 65 +++++++++++------------------ pkg/s3/nodeserver.go | 36 ++++------------ pkg/s3/s3fs.go | 43 ------------------- 8 files changed, 204 insertions(+), 157 deletions(-) delete mode 100644 pkg/s3/goofys.go create mode 100644 pkg/s3/mounter.go create mode 100644 pkg/s3/mounter_goofys.go create mode 100644 pkg/s3/mounter_s3fs.go rename pkg/s3/{s3ql.go => mounter_s3ql.go} (59%) delete mode 100644 pkg/s3/s3fs.go diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 8b2f4bc..1d3b859 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -55,20 +55,21 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } if !exists { - if err := cs.s3.client.createBucket(volumeID); err != nil { + if err = cs.s3.client.createBucket(volumeID); err != nil { glog.V(3).Infof("failed to create volume: %v", err) return nil, err } } - mounter := cs.s3.cfg.Mounter - if mounter == "" { - mounter = req.GetParameters()[mounterKey] + mounterType := cs.s3.cfg.Mounter + if mounterType == "" { + mounterType = req.GetParameters()[mounterKey] } - switch mounter { - case s3qlMounter: - if err := s3qlCreate(volumeID, cs.s3.cfg); err != nil { - return nil, err - } + mounter, err := newMounter(mounterType, volumeID, cs.s3.cfg) + if err != nil { + return nil, err + } + if err := mounter.Format(); err != nil { + return nil, err } glog.V(4).Infof("create volume %s", volumeID) diff --git a/pkg/s3/goofys.go b/pkg/s3/goofys.go deleted file mode 100644 index d28ff7c..0000000 --- a/pkg/s3/goofys.go +++ /dev/null @@ -1,37 +0,0 @@ -package s3 - -import ( - "fmt" - "os" - - "context" - - goofys "github.com/kahing/goofys/api" -) - -const defaultRegion = "us-east-1" - -func goofysMount(bucket string, cfg *Config, targetPath string) error { - goofysCfg := &goofys.Config{ - MountPoint: targetPath, - Endpoint: cfg.Endpoint, - Region: cfg.Region, - DirMode: 0755, - FileMode: 0644, - MountOptions: map[string]string{ - "allow_other": "", - }, - } - if cfg.Endpoint != "" { - cfg.Region = defaultRegion - } - os.Setenv("AWS_ACCESS_KEY_ID", cfg.AccessKeyID) - os.Setenv("AWS_SECRET_ACCESS_KEY", cfg.SecretAccessKey) - - _, _, err := goofys.Mount(context.Background(), bucket, goofysCfg) - - if err != nil { - return fmt.Errorf("Error mounting via goofys: %s", err) - } - return nil -} diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go new file mode 100644 index 0000000..55d148a --- /dev/null +++ b/pkg/s3/mounter.go @@ -0,0 +1,34 @@ +package s3 + +import "fmt" + +// Mounter interface which can be implemented +// by the different mounter types +type Mounter interface { + Format() error + Mount(targetPath string) error +} + +const ( + mounterKey = "mounter" + s3fsMounterType = "s3fs" + goofysMounterType = "goofys" + s3qlMounterType = "s3ql" +) + +// newMounter returns a new mounter depending on the mounterType parameter +func newMounter(mounterType string, bucket string, cfg *Config) (Mounter, error) { + switch mounterType { + case "": + case s3fsMounterType: + return newS3fsMounter(bucket, cfg) + + case goofysMounterType: + return newGoofysMounter(bucket, cfg) + + case s3qlMounterType: + return newS3qlMounter(bucket, cfg) + + } + return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", bucket, mounterType) +} diff --git a/pkg/s3/mounter_goofys.go b/pkg/s3/mounter_goofys.go new file mode 100644 index 0000000..2a1b8c2 --- /dev/null +++ b/pkg/s3/mounter_goofys.go @@ -0,0 +1,63 @@ +package s3 + +import ( + "fmt" + "os" + + "context" + + goofysApi "github.com/kahing/goofys/api" +) + +const defaultRegion = "us-east-1" + +// Implements Mounter +type goofysMounter struct { + bucket string + endpoint string + region string + accessKeyID string + secretAccessKey string +} + +func newGoofysMounter(bucket string, cfg *Config) (Mounter, error) { + region := cfg.Region + // if endpoint is set we need a default region + if region == "" && cfg.Endpoint != "" { + region = defaultRegion + } + return &goofysMounter{ + bucket: bucket, + endpoint: cfg.Endpoint, + region: region, + accessKeyID: cfg.AccessKeyID, + secretAccessKey: cfg.SecretAccessKey, + }, nil +} + +func (goofys *goofysMounter) Format() error { + return nil +} + +func (goofys *goofysMounter) Mount(targetPath string) error { + goofysCfg := &goofysApi.Config{ + MountPoint: targetPath, + Endpoint: goofys.endpoint, + Region: goofys.region, + DirMode: 0755, + FileMode: 0644, + MountOptions: map[string]string{ + "allow_other": "", + }, + } + + os.Setenv("AWS_ACCESS_KEY_ID", goofys.accessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", goofys.secretAccessKey) + + _, _, err := goofysApi.Mount(context.Background(), goofys.bucket, goofysCfg) + + if err != nil { + return fmt.Errorf("Error mounting via goofys: %s", err) + } + return nil +} diff --git a/pkg/s3/mounter_s3fs.go b/pkg/s3/mounter_s3fs.go new file mode 100644 index 0000000..812f644 --- /dev/null +++ b/pkg/s3/mounter_s3fs.go @@ -0,0 +1,64 @@ +package s3 + +import ( + "fmt" + "os" + "os/exec" +) + +// Implements Mounter +type s3fsMounter struct { + bucket string + url string + region string + pwFileContent string +} + +func newS3fsMounter(bucket string, cfg *Config) (Mounter, error) { + return &s3fsMounter{ + bucket: bucket, + url: cfg.Endpoint, + region: cfg.Region, + pwFileContent: cfg.AccessKeyID + ":" + cfg.SecretAccessKey, + }, nil +} + +func (s3fs *s3fsMounter) Format() error { + return nil +} + +func (s3fs *s3fsMounter) Mount(targetPath string) error { + if err := writes3fsPass(s3fs.pwFileContent); err != nil { + return err + } + args := []string{ + fmt.Sprintf("%s", s3fs.bucket), + fmt.Sprintf("%s", targetPath), + "-o", "sigv2", + "-o", "use_path_request_style", + "-o", fmt.Sprintf("url=%s", s3fs.url), + "-o", fmt.Sprintf("endpoint=%s", s3fs.region), + "-o", "allow_other", + "-o", "mp_umask=000", + } + cmd := exec.Command("s3fs", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Error mounting using s3fs, output: %s", out) + } + return nil +} + +func writes3fsPass(pwFileContent string) error { + pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) + pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return err + } + _, err = pwFile.WriteString(pwFileContent) + if err != nil { + return err + } + pwFile.Close() + return nil +} diff --git a/pkg/s3/s3ql.go b/pkg/s3/mounter_s3ql.go similarity index 59% rename from pkg/s3/s3ql.go rename to pkg/s3/mounter_s3ql.go index 3e1a7b8..4995478 100644 --- a/pkg/s3/s3ql.go +++ b/pkg/s3/mounter_s3ql.go @@ -13,7 +13,8 @@ import ( "gopkg.in/ini.v1" ) -type s3fsConfig struct { +// Implements Mounter +type s3qlMounter struct { url string bucketURL string login string @@ -29,7 +30,7 @@ const ( s3qlCmdMount = "mount.s3ql" ) -func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) { +func newS3qlMounter(bucket string, cfg *Config) (Mounter, error) { url, err := url.Parse(cfg.Endpoint) if err != nil { return nil, err @@ -38,13 +39,12 @@ func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) if strings.Contains(url.Scheme, "http") { url.Scheme = "s3c" } - s3ql := &s3fsConfig{ + s3ql := &s3qlMounter{ url: url.String(), login: cfg.AccessKeyID, password: cfg.SecretAccessKey, passphrase: cfg.EncryptionKey, ssl: ssl, - targetPath: targetPath, } url.Path = path.Join(url.Path, bucket) @@ -57,21 +57,25 @@ func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) return s3ql, s3ql.writeConfig() } -func s3qlCreate(bucket string, cfg *Config) error { - s3ql, err := newS3ql(bucket, "unknown", cfg) - if err != nil { - return err +func (s3ql *s3qlMounter) Format() error { + // force creation to ignore existing data + args := []string{ + s3ql.bucketURL, + "--force", } - return s3ql.create() + + p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase) + reader := bytes.NewReader([]byte(p)) + return s3qlCmd(s3qlCmdMkfs, append(args, s3ql.options...), reader) } -func s3qlMount(bucket string, cfg *Config, targetPath string) error { - s3ql, err := newS3ql(bucket, targetPath, cfg) - if err != nil { - return err +func (s3ql *s3qlMounter) Mount(targetPath string) error { + args := []string{ + s3ql.bucketURL, + targetPath, + "--allow-other", } - - return s3ql.mount() + return s3qlCmd(s3qlCmdMount, append(args, s3ql.options...), nil) } func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { @@ -87,38 +91,17 @@ func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { return nil } -func (cfg *s3fsConfig) create() error { - // force creation to ignore existing data - args := []string{ - cfg.bucketURL, - "--force", - } - - p := fmt.Sprintf("%s\n%s\n", cfg.passphrase, cfg.passphrase) - reader := bytes.NewReader([]byte(p)) - return s3qlCmd(s3qlCmdMkfs, append(args, cfg.options...), reader) -} - -func (cfg *s3fsConfig) mount() error { - args := []string{ - cfg.bucketURL, - cfg.targetPath, - "--allow-other", - } - return s3qlCmd(s3qlCmdMount, append(args, cfg.options...), nil) -} - -func (cfg *s3fsConfig) writeConfig() error { +func (s3ql *s3qlMounter) writeConfig() error { s3qlIni := ini.Empty() section, err := s3qlIni.NewSection("s3ql") if err != nil { return err } - section.NewKey("storage-url", cfg.url) - section.NewKey("backend-login", cfg.login) - section.NewKey("backend-password", cfg.password) - section.NewKey("fs-passphrase", cfg.passphrase) + section.NewKey("storage-url", s3ql.url) + section.NewKey("backend-login", s3ql.login) + section.NewKey("backend-password", s3ql.password) + section.NewKey("fs-passphrase", s3ql.passphrase) authDir := os.Getenv("HOME") + "/.s3ql" authFile := authDir + "/authinfo2" diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 9fd88f5..446190a 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -17,7 +17,6 @@ limitations under the License. package s3 import ( - "fmt" "os" "github.com/golang/glog" @@ -31,13 +30,6 @@ import ( "github.com/kubernetes-csi/drivers/pkg/csi-common" ) -const ( - mounterKey = "mounter" - s3fsMounter = "s3fs" - goofysMounter = "goofys" - s3qlMounter = "s3ql" -) - type nodeServer struct { *csicommon.DefaultNodeServer *s3 @@ -86,26 +78,16 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - mounter := ns.s3.cfg.Mounter - if mounter == "" { - mounter = attrib[mounterKey] + mounterType := ns.s3.cfg.Mounter + if mounterType == "" { + mounterType = attrib[mounterKey] } - switch mounter { - case "": - case s3fsMounter: - if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil { - return nil, err - } - case goofysMounter: - if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil { - return nil, err - } - case s3qlMounter: - if err := s3qlMount(volumeID, ns.s3.cfg, targetPath); err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, ns.s3.cfg.Mounter) + mounter, err := newMounter(mounterType, volumeID, ns.s3.cfg) + if err != nil { + return nil, err + } + if err := mounter.Mount(targetPath); err != nil { + return nil, err } glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath) diff --git a/pkg/s3/s3fs.go b/pkg/s3/s3fs.go deleted file mode 100644 index 1f48cb7..0000000 --- a/pkg/s3/s3fs.go +++ /dev/null @@ -1,43 +0,0 @@ -package s3 - -import ( - "fmt" - "os" - "os/exec" -) - -func s3fsMount(bucket string, cfg *Config, targetPath string) error { - if err := writes3fsPass(cfg); err != nil { - return err - } - args := []string{ - fmt.Sprintf("%s", bucket), - fmt.Sprintf("%s", targetPath), - "-o", "sigv2", - "-o", "use_path_request_style", - "-o", fmt.Sprintf("url=%s", cfg.Endpoint), - "-o", fmt.Sprintf("endpoint=%s", cfg.Region), - "-o", "allow_other", - "-o", "mp_umask=000", - } - cmd := exec.Command("s3fs", args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("Error mounting using s3fs, output: %s", out) - } - return nil -} - -func writes3fsPass(cfg *Config) error { - pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) - pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return err - } - _, err = pwFile.WriteString(cfg.AccessKeyID + ":" + cfg.SecretAccessKey) - if err != nil { - return err - } - pwFile.Close() - return nil -} From 8cd8f6b6cde449c0439b9aa6ad0cf04ccf984e5e Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 21 Jul 2018 15:23:11 +0200 Subject: [PATCH 5/8] Mounter can now only be specified through flag --- cmd/s3driver/main.go | 2 ++ deploy/kubernetes/csi-s3-driver.yaml | 6 ++++++ deploy/kubernetes/provisioner.yaml | 6 ++++++ deploy/kubernetes/secret.yaml | 8 ++++++-- deploy/kubernetes/storageclass.yaml | 5 ----- pkg/s3/controllerserver.go | 7 ++----- pkg/s3/mounter.go | 9 ++++----- pkg/s3/mounter_goofys.go | 5 +++++ pkg/s3/mounter_s3fs.go | 6 ++++++ pkg/s3/mounter_s3ql.go | 12 ++++++++++-- pkg/s3/nodeserver.go | 12 ++++++------ 11 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cmd/s3driver/main.go b/cmd/s3driver/main.go index f071b10..805804d 100644 --- a/cmd/s3driver/main.go +++ b/cmd/s3driver/main.go @@ -35,6 +35,7 @@ var ( secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use") s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use") region = flag.String("region", "", "S3 Region to use") + mounter = flag.String("mounter", "s3fs", "Specify which Mounter to use") encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)") ) @@ -46,6 +47,7 @@ func main() { SecretAccessKey: *secretAccessKey, Endpoint: *s3endpoint, Region: *region, + Mounter: *mounter, EncryptionKey: *encryptionKey, } diff --git a/deploy/kubernetes/csi-s3-driver.yaml b/deploy/kubernetes/csi-s3-driver.yaml index 931a831..3dba016 100644 --- a/deploy/kubernetes/csi-s3-driver.yaml +++ b/deploy/kubernetes/csi-s3-driver.yaml @@ -83,6 +83,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--mounter=$(MOUNTER)" - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: @@ -112,6 +113,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: MOUNTER + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: mounter - name: ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/deploy/kubernetes/provisioner.yaml b/deploy/kubernetes/provisioner.yaml index 11f9e97..99cfebc 100644 --- a/deploy/kubernetes/provisioner.yaml +++ b/deploy/kubernetes/provisioner.yaml @@ -87,6 +87,7 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--mounter=$(MOUNTER)" - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: @@ -116,6 +117,11 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: MOUNTER + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: mounter - name: ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index b0119fa..7342f52 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -6,7 +6,11 @@ stringData: accessKeyID: secretAccessKey: endpoint: - # If not on S3, just set it to "" + # If not on S3, set it to "" region: + # specify which mounter to use + # can be set to s3fs, goofys or s3ql + mounter: # Currently only for s3ql - # encryptionKey: + # If not using s3ql, set it to "" + encryptionKey: diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index a6dfd9b..dfe708d 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -4,8 +4,3 @@ apiVersion: storage.k8s.io/v1 metadata: name: csi-s3 provisioner: ch.ctrox.csi.s3-driver -parameters: - # specify which mounter to use - # can be set to s3fs, goofys or s3ql - # s3fs is the default - # mounter: s3ql diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 1d3b859..f186bf1 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -60,11 +60,8 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } } - mounterType := cs.s3.cfg.Mounter - if mounterType == "" { - mounterType = req.GetParameters()[mounterKey] - } - mounter, err := newMounter(mounterType, volumeID, cs.s3.cfg) + + mounter, err := newMounter(volumeID, cs.s3.cfg) if err != nil { return nil, err } diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go index 55d148a..aaaefd9 100644 --- a/pkg/s3/mounter.go +++ b/pkg/s3/mounter.go @@ -7,19 +7,18 @@ import "fmt" type Mounter interface { Format() error Mount(targetPath string) error + Unmount(targetPath string) error } const ( - mounterKey = "mounter" s3fsMounterType = "s3fs" goofysMounterType = "goofys" s3qlMounterType = "s3ql" ) // newMounter returns a new mounter depending on the mounterType parameter -func newMounter(mounterType string, bucket string, cfg *Config) (Mounter, error) { - switch mounterType { - case "": +func newMounter(bucket string, cfg *Config) (Mounter, error) { + switch cfg.Mounter { case s3fsMounterType: return newS3fsMounter(bucket, cfg) @@ -30,5 +29,5 @@ func newMounter(mounterType string, bucket string, cfg *Config) (Mounter, error) return newS3qlMounter(bucket, cfg) } - return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", bucket, mounterType) + return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", bucket, cfg.Mounter) } diff --git a/pkg/s3/mounter_goofys.go b/pkg/s3/mounter_goofys.go index 2a1b8c2..c3a0066 100644 --- a/pkg/s3/mounter_goofys.go +++ b/pkg/s3/mounter_goofys.go @@ -7,6 +7,7 @@ import ( "context" goofysApi "github.com/kahing/goofys/api" + "k8s.io/kubernetes/pkg/util/mount" ) const defaultRegion = "us-east-1" @@ -61,3 +62,7 @@ func (goofys *goofysMounter) Mount(targetPath string) error { } return nil } + +func (goofys *goofysMounter) Unmount(targetPath string) error { + return mount.New("").Unmount(targetPath) +} diff --git a/pkg/s3/mounter_s3fs.go b/pkg/s3/mounter_s3fs.go index 812f644..325e91d 100644 --- a/pkg/s3/mounter_s3fs.go +++ b/pkg/s3/mounter_s3fs.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "os/exec" + + "k8s.io/kubernetes/pkg/util/mount" ) // Implements Mounter @@ -49,6 +51,10 @@ func (s3fs *s3fsMounter) Mount(targetPath string) error { return nil } +func (s3fs *s3fsMounter) Unmount(targetPath string) error { + return mount.New("").Unmount(targetPath) +} + func writes3fsPass(pwFileContent string) error { pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go index 4995478..0932da3 100644 --- a/pkg/s3/mounter_s3ql.go +++ b/pkg/s3/mounter_s3ql.go @@ -26,8 +26,9 @@ type s3qlMounter struct { } const ( - s3qlCmdMkfs = "mkfs.s3ql" - s3qlCmdMount = "mount.s3ql" + s3qlCmdMkfs = "mkfs.s3ql" + s3qlCmdMount = "mount.s3ql" + s3qlCmdUnmount = "umount.s3ql" ) func newS3qlMounter(bucket string, cfg *Config) (Mounter, error) { @@ -78,6 +79,13 @@ func (s3ql *s3qlMounter) Mount(targetPath string) error { return s3qlCmd(s3qlCmdMount, append(args, s3ql.options...), nil) } +func (s3ql *s3qlMounter) Unmount(targetPath string) error { + args := []string{ + targetPath, + } + return s3qlCmd(s3qlCmdUnmount, append(args, s3ql.options...), nil) +} + func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { cmd := exec.Command(s3qlCmd, args...) if stdin != nil { diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 446190a..a974db4 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -78,11 +78,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - mounterType := ns.s3.cfg.Mounter - if mounterType == "" { - mounterType = attrib[mounterKey] - } - mounter, err := newMounter(mounterType, volumeID, ns.s3.cfg) + mounter, err := newMounter(volumeID, ns.s3.cfg) if err != nil { return nil, err } @@ -105,7 +101,11 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.InvalidArgument, "Target path missing in request") } - err := mount.New("").Unmount(req.GetTargetPath()) + mounter, err := newMounter(req.GetVolumeId(), ns.s3.cfg) + if err != nil { + return nil, err + } + mounter.Unmount(req.GetTargetPath()) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } From b412e819771fe63fcd458a1f2cb246187b41401e Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 21 Jul 2018 16:36:39 +0200 Subject: [PATCH 6/8] Fix s3ql unmount --- cmd/s3driver/Dockerfile.s3ql | 2 +- pkg/s3/mounter_s3ql.go | 5 +---- pkg/s3/nodeserver.go | 4 ++-- test/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/s3driver/Dockerfile.s3ql b/cmd/s3driver/Dockerfile.s3ql index d73f638..da52a29 100644 --- a/cmd/s3driver/Dockerfile.s3ql +++ b/cmd/s3driver/Dockerfile.s3ql @@ -5,7 +5,7 @@ ARG S3QL_VERSION=release-2.28 RUN apt-get update && \ apt-get install -y \ - s3fs wget python3 python3-setuptools \ + s3fs wget psmisc python3 python3-setuptools \ python3-dev python3-pip python3-llfuse pkg-config \ sqlite3 libsqlite3-dev python3-apsw cython && \ rm -rf /var/lib/apt/lists/* diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go index 0932da3..df4ad91 100644 --- a/pkg/s3/mounter_s3ql.go +++ b/pkg/s3/mounter_s3ql.go @@ -80,10 +80,7 @@ func (s3ql *s3qlMounter) Mount(targetPath string) error { } func (s3ql *s3qlMounter) Unmount(targetPath string) error { - args := []string{ - targetPath, - } - return s3qlCmd(s3qlCmdUnmount, append(args, s3ql.options...), nil) + return s3qlCmd(s3qlCmdUnmount, []string{targetPath}, nil) } func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index a974db4..c6dad2e 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -105,14 +105,14 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu if err != nil { return nil, err } - mounter.Unmount(req.GetTargetPath()) - if err != nil { + if err := mounter.Unmount(req.GetTargetPath()); err != nil { return nil, status.Error(codes.Internal, err.Error()) } glog.V(4).Infof("s3: bucket %s has been unmounted.", req.GetVolumeId()) return &csi.NodeUnpublishVolumeResponse{}, nil } + func (ns *nodeServer) NodeStageVolume( ctx context.Context, req *csi.NodeStageVolumeRequest) ( diff --git a/test/Dockerfile b/test/Dockerfile index 3f0a12b..c128ab4 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -5,7 +5,7 @@ ARG S3QL_VERSION=release-2.28 RUN apt-get update && \ apt-get install -y \ - s3fs wget python3 python3-setuptools \ + s3fs wget psmisc python3 python3-setuptools \ python3-dev python3-pip python3-llfuse pkg-config \ sqlite3 libsqlite3-dev python3-apsw cython && \ rm -rf /var/lib/apt/lists/* From c4880a2bdce455e5cfe1e985aa6d80638b99f773 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sat, 21 Jul 2018 17:18:19 +0200 Subject: [PATCH 7/8] Add ps util to container --- cmd/s3driver/Dockerfile.s3ql | 2 +- test/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/s3driver/Dockerfile.s3ql b/cmd/s3driver/Dockerfile.s3ql index da52a29..c631f09 100644 --- a/cmd/s3driver/Dockerfile.s3ql +++ b/cmd/s3driver/Dockerfile.s3ql @@ -5,7 +5,7 @@ ARG S3QL_VERSION=release-2.28 RUN apt-get update && \ apt-get install -y \ - s3fs wget psmisc python3 python3-setuptools \ + s3fs wget psmisc procps python3 python3-setuptools \ python3-dev python3-pip python3-llfuse pkg-config \ sqlite3 libsqlite3-dev python3-apsw cython && \ rm -rf /var/lib/apt/lists/* diff --git a/test/Dockerfile b/test/Dockerfile index c128ab4..c61c431 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -5,7 +5,7 @@ ARG S3QL_VERSION=release-2.28 RUN apt-get update && \ apt-get install -y \ - s3fs wget psmisc python3 python3-setuptools \ + s3fs wget psmisc procps python3 python3-setuptools \ python3-dev python3-pip python3-llfuse pkg-config \ sqlite3 libsqlite3-dev python3-apsw cython && \ rm -rf /var/lib/apt/lists/* From 626f451184f750c7029832f225b5043a9b5dd858 Mon Sep 17 00:00:00 2001 From: Cyrill Troxler Date: Sun, 22 Jul 2018 12:59:30 +0200 Subject: [PATCH 8/8] Update README --- README.md | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 860e027..18e5618 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # CSI for S3 This is a Container Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) for S3 (or S3 compatible) storage. This can dynamically allocate buckets and mount them via a fuse mount into any container. +# Status +This is still very experimental and should not be used in any production environment. Unexpected data loss could occur depending on what mounter and S3 storage backend is being used. + # Kubernetes installation ## Requirements * Kubernetes 1.10+ @@ -19,6 +22,12 @@ stringData: secretAccessKey: endpoint: + # specify which mounter to use + # can be set to s3fs, goofys or s3ql + mounter: + # Currently only for s3ql + # If not using s3ql, set it to "" + encryptionKey: ``` ## 2. Deploy the driver @@ -62,20 +71,32 @@ If something does not work as expected, check the troubleshooting section below. # Additional configuration ## Mounter -By default the driver will use [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) to mount buckets. Alternatively you can configure the storage class to use [goofys](https://github.com/kahing/goofys) for mounting S3 buckets. Note that goofys has some drawbacks in regards to POSIX compliance but in return offers better Performance than s3fs. +As seen in the deployment example above, the driver can be configured to use one of these mounters to mount buckets: -To configure a storage class to use goofys, just set the `mounter` parameter to `goofys` -```yaml -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: csi-s3 -provisioner: ch.ctrox.csi.s3-driver -parameters: - mounter: goofys - csiProvisionerSecretName: csi-s3-secret - csiProvisionerSecretNamespace: kube-system -``` +* [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) +* [goofys](https://github.com/kahing/goofys) +* [s3ql](https://github.com/s3ql/s3ql) + +All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter: + +### s3fs +* Large subset of POSIX +* Files can be viewed normally with any S3 client +* Does not support appends or random writes + +### goofys +* Weak POSIX compatibility +* Performance first +* Files can be viewed normally with any S3 client +* Does not support appends or random writes + +### s3ql +* (Almost) full POSIX compatibility +* Uses its own object format +* Files are not readable with other S3 clients +* Support appends +* Supports compression before upload +* Supports encryption before upload # Limitations As S3 is not a real file system there are some limitations to consider here. Depending on what mounter you are using, you will have different levels of POSIX compability. Also depending on what S3 storage backend you are using there are not always consistency guarantees. The detailed limitations can be found on the documentation of [s3fs](https://github.com/s3fs-fuse/s3fs-fuse#limitations) and [goofys](https://github.com/kahing/goofys#current-status).