diff --git a/Gopkg.lock b/Gopkg.lock index 4002686..d56885f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -125,13 +125,13 @@ version = "v0.19.0" [[projects]] - branch = "master" name = "github.com/kubernetes-csi/csi-test" packages = [ "pkg/sanity", "utils" ] - revision = "6fed82d24d3a04c1814440d4178e7bf8ae9e67e6" + revision = "718c9544f5e16cba31881333e0f9dff371663dea" + version = "v0.3.0-1" [[projects]] branch = "master" @@ -334,6 +334,12 @@ revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" version = "v1.13.0" +[[projects]] + name = "gopkg.in/ini.v1" + packages = ["."] + revision = "358ee7663966325963d4e8b2e1fbd570c5195153" + version = "v1.38.1" + [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] @@ -366,6 +372,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4" + inputs-digest = "659f47734b56af7fba146039231841e82cc4c3c95dff2814ec3688e967790a50" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1a7f0e9..98f9a1b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -11,8 +11,8 @@ version = "0.19.0" [[constraint]] - branch = "master" name = "github.com/kubernetes-csi/csi-test" + version = "v0.3.0-1" [[constraint]] branch = "master" @@ -37,3 +37,7 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "gopkg.in/ini.v1" + version = "1.38.1" diff --git a/Makefile b/Makefile index 7c9b476..1b55be7 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver REGISTRY_NAME=ctrox IMAGE_NAME=csi-s3-driver -IMAGE_VERSION=0.1.0 +IMAGE_VERSION=0.2.0 IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION) TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test @@ -27,7 +27,7 @@ test: docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile . docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG) container: build - docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile . + docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql . push: container docker push $(IMAGE_TAG) clean: diff --git a/README.md b/README.md index 860e027..18e5618 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # CSI for S3 This is a Container Storage Interface ([CSI](https://github.com/container-storage-interface/spec/blob/master/spec.md)) for S3 (or S3 compatible) storage. This can dynamically allocate buckets and mount them via a fuse mount into any container. +# Status +This is still very experimental and should not be used in any production environment. Unexpected data loss could occur depending on what mounter and S3 storage backend is being used. + # Kubernetes installation ## Requirements * Kubernetes 1.10+ @@ -19,6 +22,12 @@ stringData: secretAccessKey: endpoint: + # specify which mounter to use + # can be set to s3fs, goofys or s3ql + mounter: + # Currently only for s3ql + # If not using s3ql, set it to "" + encryptionKey: ``` ## 2. Deploy the driver @@ -62,20 +71,32 @@ If something does not work as expected, check the troubleshooting section below. # Additional configuration ## Mounter -By default the driver will use [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) to mount buckets. Alternatively you can configure the storage class to use [goofys](https://github.com/kahing/goofys) for mounting S3 buckets. Note that goofys has some drawbacks in regards to POSIX compliance but in return offers better Performance than s3fs. +As seen in the deployment example above, the driver can be configured to use one of these mounters to mount buckets: -To configure a storage class to use goofys, just set the `mounter` parameter to `goofys` -```yaml -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: csi-s3 -provisioner: ch.ctrox.csi.s3-driver -parameters: - mounter: goofys - csiProvisionerSecretName: csi-s3-secret - csiProvisionerSecretNamespace: kube-system -``` +* [s3fs](https://github.com/s3fs-fuse/s3fs-fuse) +* [goofys](https://github.com/kahing/goofys) +* [s3ql](https://github.com/s3ql/s3ql) + +All mounters have different strengths and weaknesses depending on your use case. Here are some characteristics which should help you choose a mounter: + +### s3fs +* Large subset of POSIX +* Files can be viewed normally with any S3 client +* Does not support appends or random writes + +### goofys +* Weak POSIX compatibility +* Performance first +* Files can be viewed normally with any S3 client +* Does not support appends or random writes + +### s3ql +* (Almost) full POSIX compatibility +* Uses its own object format +* Files are not readable with other S3 clients +* Support appends +* Supports compression before upload +* Supports encryption before upload # Limitations As S3 is not a real file system there are some limitations to consider here. Depending on what mounter you are using, you will have different levels of POSIX compability. Also depending on what S3 storage backend you are using there are not always consistency guarantees. The detailed limitations can be found on the documentation of [s3fs](https://github.com/s3fs-fuse/s3fs-fuse#limitations) and [goofys](https://github.com/kahing/goofys#current-status). diff --git a/cmd/s3driver/Dockerfile.s3ql b/cmd/s3driver/Dockerfile.s3ql new file mode 100644 index 0000000..c631f09 --- /dev/null +++ b/cmd/s3driver/Dockerfile.s3ql @@ -0,0 +1,23 @@ +FROM debian:stretch +LABEL maintainers="Cyrill Troxler " +LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget psmisc procps python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install + +COPY ./_output/s3driver /s3driver +ENTRYPOINT ["/s3driver"] diff --git a/cmd/s3driver/main.go b/cmd/s3driver/main.go index 1d9eee7..805804d 100644 --- a/cmd/s3driver/main.go +++ b/cmd/s3driver/main.go @@ -35,6 +35,8 @@ var ( secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use") s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use") region = flag.String("region", "", "S3 Region to use") + mounter = flag.String("mounter", "s3fs", "Specify which Mounter to use") + encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)") ) func main() { @@ -45,6 +47,8 @@ func main() { SecretAccessKey: *secretAccessKey, Endpoint: *s3endpoint, Region: *region, + Mounter: *mounter, + EncryptionKey: *encryptionKey, } driver, err := s3.NewS3(*nodeID, *endpoint, cfg) diff --git a/deploy/kubernetes/csi-s3-driver.yaml b/deploy/kubernetes/csi-s3-driver.yaml index cc8443a..3dba016 100644 --- a/deploy/kubernetes/csi-s3-driver.yaml +++ b/deploy/kubernetes/csi-s3-driver.yaml @@ -75,7 +75,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -83,6 +83,8 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--mounter=$(MOUNTER)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -111,6 +113,16 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: MOUNTER + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: mounter + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: plugin-dir diff --git a/deploy/kubernetes/provisioner.yaml b/deploy/kubernetes/provisioner.yaml index 1be1b5d..99cfebc 100644 --- a/deploy/kubernetes/provisioner.yaml +++ b/deploy/kubernetes/provisioner.yaml @@ -79,7 +79,7 @@ spec: - name: socket-dir mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver - name: s3-csi-driver - image: ctrox/csi-s3-driver:0.1.0 + image: ctrox/csi-s3-driver:0.2.0 args: - "--endpoint=$(CSI_ENDPOINT)" - "--nodeid=$(NODE_ID)" @@ -87,6 +87,8 @@ spec: - "--secret-access-key=$(SECRET_ACCESS_KEY)" - "--s3-endpoint=$(S3_ENDPOINT)" - "--region=$(REGION)" + - "--mounter=$(MOUNTER)" + - "--encryption-key=$(ENCRYPTION_KEY)" - "--v=4" env: - name: CSI_ENDPOINT @@ -115,6 +117,16 @@ spec: secretKeyRef: name: csi-s3-secret key: region + - name: MOUNTER + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: mounter + - name: ENCRYPTION_KEY + valueFrom: + secretKeyRef: + name: csi-s3-secret + key: encryptionKey imagePullPolicy: "Always" volumeMounts: - name: socket-dir diff --git a/deploy/kubernetes/pvc.yaml b/deploy/kubernetes/pvc.yaml index a645b4e..68f0496 100644 --- a/deploy/kubernetes/pvc.yaml +++ b/deploy/kubernetes/pvc.yaml @@ -5,7 +5,7 @@ metadata: namespace: default spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/deploy/kubernetes/secret.yaml b/deploy/kubernetes/secret.yaml index 266157c..7342f52 100644 --- a/deploy/kubernetes/secret.yaml +++ b/deploy/kubernetes/secret.yaml @@ -6,5 +6,11 @@ stringData: accessKeyID: secretAccessKey: endpoint: - # If not on S3, just set it to "" + # If not on S3, set it to "" region: + # specify which mounter to use + # can be set to s3fs, goofys or s3ql + mounter: + # Currently only for s3ql + # If not using s3ql, set it to "" + encryptionKey: diff --git a/deploy/kubernetes/storageclass.yaml b/deploy/kubernetes/storageclass.yaml index 3db2aef..dfe708d 100644 --- a/deploy/kubernetes/storageclass.yaml +++ b/deploy/kubernetes/storageclass.yaml @@ -4,8 +4,3 @@ apiVersion: storage.k8s.io/v1 metadata: name: csi-s3 provisioner: ch.ctrox.csi.s3-driver -parameters: - # specify which mounter to use - # can be set to s3fs or goofys - # s3fs is the default - # mounter: s3fs diff --git a/pkg/s3/config.go b/pkg/s3/config.go index 43eecc0..f5b1714 100644 --- a/pkg/s3/config.go +++ b/pkg/s3/config.go @@ -6,4 +6,6 @@ type Config struct { SecretAccessKey string Region string Endpoint string + Mounter string + EncryptionKey string } diff --git a/pkg/s3/controllerserver.go b/pkg/s3/controllerserver.go index 9f2c4f7..f186bf1 100644 --- a/pkg/s3/controllerserver.go +++ b/pkg/s3/controllerserver.go @@ -55,12 +55,20 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } if !exists { - if err := cs.s3.client.createBucket(volumeID); err != nil { + if err = cs.s3.client.createBucket(volumeID); err != nil { glog.V(3).Infof("failed to create volume: %v", err) return nil, err } } + mounter, err := newMounter(volumeID, cs.s3.cfg) + if err != nil { + return nil, err + } + if err := mounter.Format(); err != nil { + return nil, err + } + glog.V(4).Infof("create volume %s", volumeID) s3Vol := s3Volume{} s3Vol.VolName = req.GetName() @@ -68,7 +76,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ Id: volumeID, - CapacityBytes: 0, + CapacityBytes: 1, Attributes: req.GetParameters(), }, }, nil @@ -123,6 +131,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId())) } - // We currently support all capabilities - return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil + for _, cap := range req.VolumeCapabilities { + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil + } + } + return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil } diff --git a/pkg/s3/goofys.go b/pkg/s3/goofys.go deleted file mode 100644 index d28ff7c..0000000 --- a/pkg/s3/goofys.go +++ /dev/null @@ -1,37 +0,0 @@ -package s3 - -import ( - "fmt" - "os" - - "context" - - goofys "github.com/kahing/goofys/api" -) - -const defaultRegion = "us-east-1" - -func goofysMount(bucket string, cfg *Config, targetPath string) error { - goofysCfg := &goofys.Config{ - MountPoint: targetPath, - Endpoint: cfg.Endpoint, - Region: cfg.Region, - DirMode: 0755, - FileMode: 0644, - MountOptions: map[string]string{ - "allow_other": "", - }, - } - if cfg.Endpoint != "" { - cfg.Region = defaultRegion - } - os.Setenv("AWS_ACCESS_KEY_ID", cfg.AccessKeyID) - os.Setenv("AWS_SECRET_ACCESS_KEY", cfg.SecretAccessKey) - - _, _, err := goofys.Mount(context.Background(), bucket, goofysCfg) - - if err != nil { - return fmt.Errorf("Error mounting via goofys: %s", err) - } - return nil -} diff --git a/pkg/s3/mounter.go b/pkg/s3/mounter.go new file mode 100644 index 0000000..aaaefd9 --- /dev/null +++ b/pkg/s3/mounter.go @@ -0,0 +1,33 @@ +package s3 + +import "fmt" + +// Mounter interface which can be implemented +// by the different mounter types +type Mounter interface { + Format() error + Mount(targetPath string) error + Unmount(targetPath string) error +} + +const ( + s3fsMounterType = "s3fs" + goofysMounterType = "goofys" + s3qlMounterType = "s3ql" +) + +// newMounter returns a new mounter depending on the mounterType parameter +func newMounter(bucket string, cfg *Config) (Mounter, error) { + switch cfg.Mounter { + case s3fsMounterType: + return newS3fsMounter(bucket, cfg) + + case goofysMounterType: + return newGoofysMounter(bucket, cfg) + + case s3qlMounterType: + return newS3qlMounter(bucket, cfg) + + } + return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", bucket, cfg.Mounter) +} diff --git a/pkg/s3/mounter_goofys.go b/pkg/s3/mounter_goofys.go new file mode 100644 index 0000000..c3a0066 --- /dev/null +++ b/pkg/s3/mounter_goofys.go @@ -0,0 +1,68 @@ +package s3 + +import ( + "fmt" + "os" + + "context" + + goofysApi "github.com/kahing/goofys/api" + "k8s.io/kubernetes/pkg/util/mount" +) + +const defaultRegion = "us-east-1" + +// Implements Mounter +type goofysMounter struct { + bucket string + endpoint string + region string + accessKeyID string + secretAccessKey string +} + +func newGoofysMounter(bucket string, cfg *Config) (Mounter, error) { + region := cfg.Region + // if endpoint is set we need a default region + if region == "" && cfg.Endpoint != "" { + region = defaultRegion + } + return &goofysMounter{ + bucket: bucket, + endpoint: cfg.Endpoint, + region: region, + accessKeyID: cfg.AccessKeyID, + secretAccessKey: cfg.SecretAccessKey, + }, nil +} + +func (goofys *goofysMounter) Format() error { + return nil +} + +func (goofys *goofysMounter) Mount(targetPath string) error { + goofysCfg := &goofysApi.Config{ + MountPoint: targetPath, + Endpoint: goofys.endpoint, + Region: goofys.region, + DirMode: 0755, + FileMode: 0644, + MountOptions: map[string]string{ + "allow_other": "", + }, + } + + os.Setenv("AWS_ACCESS_KEY_ID", goofys.accessKeyID) + os.Setenv("AWS_SECRET_ACCESS_KEY", goofys.secretAccessKey) + + _, _, err := goofysApi.Mount(context.Background(), goofys.bucket, goofysCfg) + + if err != nil { + return fmt.Errorf("Error mounting via goofys: %s", err) + } + return nil +} + +func (goofys *goofysMounter) Unmount(targetPath string) error { + return mount.New("").Unmount(targetPath) +} diff --git a/pkg/s3/mounter_s3fs.go b/pkg/s3/mounter_s3fs.go new file mode 100644 index 0000000..325e91d --- /dev/null +++ b/pkg/s3/mounter_s3fs.go @@ -0,0 +1,70 @@ +package s3 + +import ( + "fmt" + "os" + "os/exec" + + "k8s.io/kubernetes/pkg/util/mount" +) + +// Implements Mounter +type s3fsMounter struct { + bucket string + url string + region string + pwFileContent string +} + +func newS3fsMounter(bucket string, cfg *Config) (Mounter, error) { + return &s3fsMounter{ + bucket: bucket, + url: cfg.Endpoint, + region: cfg.Region, + pwFileContent: cfg.AccessKeyID + ":" + cfg.SecretAccessKey, + }, nil +} + +func (s3fs *s3fsMounter) Format() error { + return nil +} + +func (s3fs *s3fsMounter) Mount(targetPath string) error { + if err := writes3fsPass(s3fs.pwFileContent); err != nil { + return err + } + args := []string{ + fmt.Sprintf("%s", s3fs.bucket), + fmt.Sprintf("%s", targetPath), + "-o", "sigv2", + "-o", "use_path_request_style", + "-o", fmt.Sprintf("url=%s", s3fs.url), + "-o", fmt.Sprintf("endpoint=%s", s3fs.region), + "-o", "allow_other", + "-o", "mp_umask=000", + } + cmd := exec.Command("s3fs", args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Error mounting using s3fs, output: %s", out) + } + return nil +} + +func (s3fs *s3fsMounter) Unmount(targetPath string) error { + return mount.New("").Unmount(targetPath) +} + +func writes3fsPass(pwFileContent string) error { + pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) + pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return err + } + _, err = pwFile.WriteString(pwFileContent) + if err != nil { + return err + } + pwFile.Close() + return nil +} diff --git a/pkg/s3/mounter_s3ql.go b/pkg/s3/mounter_s3ql.go new file mode 100644 index 0000000..df4ad91 --- /dev/null +++ b/pkg/s3/mounter_s3ql.go @@ -0,0 +1,117 @@ +package s3 + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path" + "strings" + + "gopkg.in/ini.v1" +) + +// Implements Mounter +type s3qlMounter struct { + url string + bucketURL string + login string + password string + passphrase string + options []string + ssl bool + targetPath string +} + +const ( + s3qlCmdMkfs = "mkfs.s3ql" + s3qlCmdMount = "mount.s3ql" + s3qlCmdUnmount = "umount.s3ql" +) + +func newS3qlMounter(bucket string, cfg *Config) (Mounter, error) { + url, err := url.Parse(cfg.Endpoint) + if err != nil { + return nil, err + } + ssl := url.Scheme != "http" + if strings.Contains(url.Scheme, "http") { + url.Scheme = "s3c" + } + s3ql := &s3qlMounter{ + url: url.String(), + login: cfg.AccessKeyID, + password: cfg.SecretAccessKey, + passphrase: cfg.EncryptionKey, + ssl: ssl, + } + + url.Path = path.Join(url.Path, bucket) + s3ql.bucketURL = url.String() + + if !ssl { + s3ql.options = []string{"--backend-options", "no-ssl"} + } + + return s3ql, s3ql.writeConfig() +} + +func (s3ql *s3qlMounter) Format() error { + // force creation to ignore existing data + args := []string{ + s3ql.bucketURL, + "--force", + } + + p := fmt.Sprintf("%s\n%s\n", s3ql.passphrase, s3ql.passphrase) + reader := bytes.NewReader([]byte(p)) + return s3qlCmd(s3qlCmdMkfs, append(args, s3ql.options...), reader) +} + +func (s3ql *s3qlMounter) Mount(targetPath string) error { + args := []string{ + s3ql.bucketURL, + targetPath, + "--allow-other", + } + return s3qlCmd(s3qlCmdMount, append(args, s3ql.options...), nil) +} + +func (s3ql *s3qlMounter) Unmount(targetPath string) error { + return s3qlCmd(s3qlCmdUnmount, []string{targetPath}, nil) +} + +func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error { + cmd := exec.Command(s3qlCmd, args...) + if stdin != nil { + cmd.Stdin = stdin + } + + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Error running s3ql command: %s", out) + } + return nil +} + +func (s3ql *s3qlMounter) writeConfig() error { + s3qlIni := ini.Empty() + section, err := s3qlIni.NewSection("s3ql") + if err != nil { + return err + } + + section.NewKey("storage-url", s3ql.url) + section.NewKey("backend-login", s3ql.login) + section.NewKey("backend-password", s3ql.password) + section.NewKey("fs-passphrase", s3ql.passphrase) + + authDir := os.Getenv("HOME") + "/.s3ql" + authFile := authDir + "/authinfo2" + os.Mkdir(authDir, 0700) + s3qlIni.SaveTo(authFile) + os.Chmod(authFile, 0600) + return nil +} diff --git a/pkg/s3/nodeserver.go b/pkg/s3/nodeserver.go index 5b7f28f..c6dad2e 100644 --- a/pkg/s3/nodeserver.go +++ b/pkg/s3/nodeserver.go @@ -17,7 +17,6 @@ limitations under the License. package s3 import ( - "fmt" "os" "github.com/golang/glog" @@ -31,12 +30,6 @@ import ( "github.com/kubernetes-csi/drivers/pkg/csi-common" ) -const ( - mounterKey = "mounter" - s3fsMounter = "s3fs" - goofysMounter = "goofys" -) - type nodeServer struct { *csicommon.DefaultNodeServer *s3 @@ -85,17 +78,12 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n", targetPath, deviceID, readOnly, volumeID, attrib, mountFlags) - mounter, exists := attrib[mounterKey] - if !exists || mounter == s3fsMounter { - if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil { - return nil, err - } - } else if mounter == goofysMounter { - if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil { - return nil, err - } - } else { - return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter) + mounter, err := newMounter(volumeID, ns.s3.cfg) + if err != nil { + return nil, err + } + if err := mounter.Mount(targetPath); err != nil { + return nil, err } glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath) @@ -113,8 +101,11 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return nil, status.Error(codes.InvalidArgument, "Target path missing in request") } - err := mount.New("").Unmount(req.GetTargetPath()) + mounter, err := newMounter(req.GetVolumeId(), ns.s3.cfg) if err != nil { + return nil, err + } + if err := mounter.Unmount(req.GetTargetPath()); err != nil { return nil, status.Error(codes.Internal, err.Error()) } glog.V(4).Infof("s3: bucket %s has been unmounted.", req.GetVolumeId()) @@ -122,28 +113,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return &csi.NodeUnpublishVolumeResponse{}, nil } -func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeStageVolumeResponse{}, nil +func (ns *nodeServer) NodeStageVolume( + ctx context.Context, + req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } -func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - - // Check arguments - if len(req.GetVolumeId()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") - } - if len(req.GetStagingTargetPath()) == 0 { - return nil, status.Error(codes.InvalidArgument, "Target path missing in request") - } - - return &csi.NodeUnstageVolumeResponse{}, nil +func (ns *nodeServer) NodeUnstageVolume( + ctx context.Context, + req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + return nil, status.Error(codes.Unimplemented, "") } diff --git a/pkg/s3/s3-client.go b/pkg/s3/s3-client.go index 7fff9a4..76713a3 100644 --- a/pkg/s3/s3-client.go +++ b/pkg/s3/s3-client.go @@ -17,10 +17,6 @@ type s3Client struct { minio *minio.Client } -type bucketMetadata struct { - CapacityBytes int64 -} - func newS3Client(cfg *Config) (*s3Client, error) { var client = &s3Client{} diff --git a/pkg/s3/s3-driver_suite_test.go b/pkg/s3/s3-driver_suite_test.go new file mode 100644 index 0000000..47b3ce6 --- /dev/null +++ b/pkg/s3/s3-driver_suite_test.go @@ -0,0 +1,116 @@ +package s3_test + +import ( + "io/ioutil" + "log" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/ctrox/csi-s3-driver/pkg/s3" + "github.com/kubernetes-csi/csi-test/pkg/sanity" +) + +const () + +var _ = Describe("S3Driver", func() { + mntDir, err := ioutil.TempDir("", "mnt") + if err != nil { + Expect(err).NotTo(HaveOccurred()) + } + + AfterSuite(func() { + os.RemoveAll(mntDir) + }) + + Context("goofys", func() { + socket := "/tmp/csi-goofys.sock" + csiEndpoint := "unix://" + socket + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "goofys", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + + Context("s3fs", func() { + socket := "/tmp/csi-s3fs.sock" + csiEndpoint := "unix://" + socket + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "s3fs", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + defer os.RemoveAll(mntDir) + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + + Context("s3ql", func() { + socket := "/tmp/csi-s3ql.sock" + csiEndpoint := "unix://" + socket + + cfg := &s3.Config{ + AccessKeyID: "FJDSJ", + SecretAccessKey: "DSG643HGDS", + Endpoint: "http://127.0.0.1:9000", + Mounter: "s3ql", + } + if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { + Expect(err).NotTo(HaveOccurred()) + } + driver, err := s3.NewS3("test-node", csiEndpoint, cfg) + if err != nil { + log.Fatal(err) + } + go driver.Run() + + defer os.RemoveAll(mntDir) + + Describe("CSI sanity", func() { + sanityCfg := &sanity.Config{ + TargetPath: mntDir, + Address: csiEndpoint, + TestVolumeSize: 1, + } + sanity.GinkgoTest(sanityCfg) + }) + }) + +}) diff --git a/pkg/s3/s3-driver_test.go b/pkg/s3/s3-driver_test.go index 4a8e875..5588247 100644 --- a/pkg/s3/s3-driver_test.go +++ b/pkg/s3/s3-driver_test.go @@ -17,42 +17,13 @@ limitations under the License. package s3 import ( - "io/ioutil" - "log" - "os" "testing" - "github.com/kubernetes-csi/csi-test/pkg/sanity" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) -func TestDriver(t *testing.T) { - socket := "/tmp/csi.sock" - endpoint := "unix://" + socket - - if err := os.Remove(socket); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to remove unix domain socket file %s, error: %s", socket, err) - } - cfg := &Config{ - AccessKeyID: "FJDSJ", - SecretAccessKey: "DSG643HGDS", - Endpoint: "http://127.0.0.1:9000", - } - driver, err := NewS3("test-node", endpoint, cfg) - if err != nil { - log.Fatal(err) - } - go driver.Run() - - mntDir, err := ioutil.TempDir("", "mnt") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(mntDir) - - sanityCfg := &sanity.Config{ - TargetPath: mntDir, - Address: endpoint, - } - - sanity.Test(t, sanityCfg) +func TestS3Driver(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "S3Driver") } diff --git a/pkg/s3/s3fs.go b/pkg/s3/s3fs.go deleted file mode 100644 index 1f48cb7..0000000 --- a/pkg/s3/s3fs.go +++ /dev/null @@ -1,43 +0,0 @@ -package s3 - -import ( - "fmt" - "os" - "os/exec" -) - -func s3fsMount(bucket string, cfg *Config, targetPath string) error { - if err := writes3fsPass(cfg); err != nil { - return err - } - args := []string{ - fmt.Sprintf("%s", bucket), - fmt.Sprintf("%s", targetPath), - "-o", "sigv2", - "-o", "use_path_request_style", - "-o", fmt.Sprintf("url=%s", cfg.Endpoint), - "-o", fmt.Sprintf("endpoint=%s", cfg.Region), - "-o", "allow_other", - "-o", "mp_umask=000", - } - cmd := exec.Command("s3fs", args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("Error mounting using s3fs, output: %s", out) - } - return nil -} - -func writes3fsPass(cfg *Config) error { - pwFileName := fmt.Sprintf("%s/.passwd-s3fs", os.Getenv("HOME")) - pwFile, err := os.OpenFile(pwFileName, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return err - } - _, err = pwFile.WriteString(cfg.AccessKeyID + ":" + cfg.SecretAccessKey) - if err != nil { - return err - } - pwFile.Close() - return nil -} diff --git a/test/Dockerfile b/test/Dockerfile index f586156..c61c431 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,8 +1,24 @@ FROM golang:stretch LABEL maintainers="Cyrill Troxler " LABEL description="s3 fuse csi plugin" +ARG S3QL_VERSION=release-2.28 + +RUN apt-get update && \ + apt-get install -y \ + s3fs wget psmisc procps python3 python3-setuptools \ + python3-dev python3-pip python3-llfuse pkg-config \ + sqlite3 libsqlite3-dev python3-apsw cython && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install defusedxml dugong requests pycrypto + +WORKDIR /usr/src +RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz +RUN tar -xzf ${S3QL_VERSION}.tar.gz +WORKDIR /usr/src/s3ql-${S3QL_VERSION} +RUN python3 setup.py build_cython build_ext --inplace +RUN python3 setup.py install -RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/* RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"]