infrastructure/k8s-csi-s3
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity

Add experimental s3ql mounter

Browse source
This commit is contained in:
Cyrill Troxler 2018-07-16 22:27:45 +02:00
parent a165937559
commit 13eba47da6
17 changed files with 247 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
Gopkg.lock generated
View file

@ -334,6 +334,12 @@
revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
version = "v1.13.0"
[[projects]]
name = "gopkg.in/ini.v1"
packages = ["."]
revision = "358ee7663966325963d4e8b2e1fbd570c5195153"
version = "v1.38.1"
[[projects]]
name = "gopkg.in/yaml.v2"
packages = ["."]
@ -366,6 +372,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "6af116857c3619ed6bf4a8c17b479733db2897293cb13de205ece61f7726b2f4"
inputs-digest = "9bd4175acb8ce47fe57c3d859bc6b061a6c5dd3017f57777f3fefb27d0020d75"
solver-name = "gps-cdcl"
solver-version = 1

4
Gopkg.toml
View file

@ -37,3 +37,7 @@
[prune]
go-tests = true
unused-packages = true
[[constraint]]
name = "gopkg.in/ini.v1"
version = "1.38.1"

4
Makefile
View file

@ -16,7 +16,7 @@
PROJECT_DIR=/go/src/github.com/ctrox/csi-s3-driver
REGISTRY_NAME=ctrox
IMAGE_NAME=csi-s3-driver
IMAGE_VERSION=0.1.0
IMAGE_VERSION=0.2.0
IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_VERSION)
TEST_IMAGE_TAG=$(REGISTRY_NAME)/$(IMAGE_NAME):test
@ -27,7 +27,7 @@ test:
docker build -t $(TEST_IMAGE_TAG) -f test/Dockerfile .
docker run --rm --privileged -v $(PWD):$(PROJECT_DIR):ro -v /dev:/dev $(TEST_IMAGE_TAG)
container: build
docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile .
docker build -t $(IMAGE_TAG) -f cmd/s3driver/Dockerfile.s3ql .
push: container
docker push $(IMAGE_TAG)
clean:

23
cmd/s3driver/Dockerfile.s3ql Normal file
View file

@ -0,0 +1,23 @@
FROM debian:stretch
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
LABEL description="s3 fuse csi plugin"
ARG S3QL_VERSION=release-2.28
RUN apt-get update && \
apt-get install -y \
s3fs wget python3 python3-setuptools \
python3-dev python3-pip python3-llfuse pkg-config \
sqlite3 libsqlite3-dev python3-apsw cython && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install defusedxml dugong requests pycrypto
WORKDIR /usr/src
RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
RUN tar -xzf ${S3QL_VERSION}.tar.gz
WORKDIR /usr/src/s3ql-${S3QL_VERSION}
RUN python3 setup.py build_cython build_ext --inplace
RUN python3 setup.py install
COPY ./_output/s3driver /s3driver
ENTRYPOINT ["/s3driver"]

2
cmd/s3driver/main.go
View file

@ -35,6 +35,7 @@ var (
secretAccessKey = flag.String("secret-access-key", "", "S3 Secret Access Key to use")
s3endpoint = flag.String("s3-endpoint", "", "S3 Endpoint URL to use")
region = flag.String("region", "", "S3 Region to use")
encryptionKey = flag.String("encryption-key", "", "Encryption key for file system (only used with s3ql)")
)
func main() {
@ -45,6 +46,7 @@ func main() {
SecretAccessKey: *secretAccessKey,
Endpoint: *s3endpoint,
Region: *region,
EncryptionKey: *encryptionKey,
}
driver, err := s3.NewS3(*nodeID, *endpoint, cfg)

8
deploy/kubernetes/csi-s3-driver.yaml
View file

@ -75,7 +75,7 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: ctrox/csi-s3-driver:0.1.0
image: ctrox/csi-s3-driver:0.2.0
args:
- "--endpoint=$(CSI_ENDPOINT)"
- "--nodeid=$(NODE_ID)"
@ -83,6 +83,7 @@ spec:
- "--secret-access-key=$(SECRET_ACCESS_KEY)"
- "--s3-endpoint=$(S3_ENDPOINT)"
- "--region=$(REGION)"
- "--encryption-key=$(ENCRYPTION_KEY)"
- "--v=4"
env:
- name: CSI_ENDPOINT
@ -111,6 +112,11 @@ spec:
secretKeyRef:
name: csi-s3-secret
key: region
- name: ENCRYPTION_KEY
valueFrom:
secretKeyRef:
name: csi-s3-secret
key: encryptionKey
imagePullPolicy: "Always"
volumeMounts:
- name: plugin-dir

8
deploy/kubernetes/provisioner.yaml
View file

@ -79,7 +79,7 @@ spec:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/ch.ctrox.csi.s3-driver
- name: s3-csi-driver
image: ctrox/csi-s3-driver:0.1.0
image: ctrox/csi-s3-driver:0.2.0
args:
- "--endpoint=$(CSI_ENDPOINT)"
- "--nodeid=$(NODE_ID)"
@ -87,6 +87,7 @@ spec:
- "--secret-access-key=$(SECRET_ACCESS_KEY)"
- "--s3-endpoint=$(S3_ENDPOINT)"
- "--region=$(REGION)"
- "--encryption-key=$(ENCRYPTION_KEY)"
- "--v=4"
env:
- name: CSI_ENDPOINT
@ -115,6 +116,11 @@ spec:
secretKeyRef:
name: csi-s3-secret
key: region
- name: ENCRYPTION_KEY
valueFrom:
secretKeyRef:
name: csi-s3-secret
key: encryptionKey
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir

2
deploy/kubernetes/pvc.yaml
View file

@ -5,7 +5,7 @@ metadata:
namespace: default
spec:
accessModes:
- ReadWriteMany
- ReadWriteOnce
resources:
requests:
storage: 5Gi

2
deploy/kubernetes/secret.yaml
View file

@ -8,3 +8,5 @@ stringData:
endpoint: <S3_ENDPOINT_URL>
# If not on S3, just set it to ""
region: <S3_REGION>
# Currently only for s3ql
# encryptionKey: <FS encryption key>

4
deploy/kubernetes/storageclass.yaml
View file

@ -6,6 +6,6 @@ metadata:
provisioner: ch.ctrox.csi.s3-driver
parameters:
# specify which mounter to use
# can be set to s3fs or goofys
# can be set to s3fs, goofys or s3ql
# s3fs is the default
# mounter: s3fs
# mounter: s3ql

2
pkg/s3/config.go
View file

@ -6,4 +6,6 @@ type Config struct {
SecretAccessKey string
Region string
Endpoint string
Mounter string
EncryptionKey string
}

18
pkg/s3/controllerserver.go
View file

@ -60,6 +60,16 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
return nil, err
}
}
mounter := cs.s3.cfg.Mounter
if mounter == "" {
mounter = req.GetParameters()[mounterKey]
}
switch mounter {
case s3qlMounter:
if err := s3qlCreate(volumeID, cs.s3.cfg); err != nil {
return nil, err
}
}
glog.V(4).Infof("create volume %s", volumeID)
s3Vol := s3Volume{}
@ -123,6 +133,10 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
return nil, status.Error(codes.NotFound, fmt.Sprintf("Volume with id %s does not exist", req.GetVolumeId()))
}
// We currently support all capabilities
return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil
for _, cap := range req.VolumeCapabilities {
if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
}
}
return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil
}

53
pkg/s3/nodeserver.go
View file

@ -35,6 +35,7 @@ const (
mounterKey = "mounter"
s3fsMounter = "s3fs"
goofysMounter = "goofys"
s3qlMounter = "s3ql"
)
type nodeServer struct {
@ -85,17 +86,26 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
glog.V(4).Infof("target %v\ndevice %v\nreadonly %v\nvolumeId %v\nattributes %v\nmountflags %v\n",
targetPath, deviceID, readOnly, volumeID, attrib, mountFlags)
mounter, exists := attrib[mounterKey]
if !exists || mounter == s3fsMounter {
mounter := ns.s3.cfg.Mounter
if mounter == "" {
mounter = attrib[mounterKey]
}
switch mounter {
case "":
case s3fsMounter:
if err := s3fsMount(volumeID, ns.s3.cfg, targetPath); err != nil {
return nil, err
}
} else if mounter == goofysMounter {
case goofysMounter:
if err := goofysMount(volumeID, ns.s3.cfg, targetPath); err != nil {
return nil, err
}
} else {
return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, mounter)
case s3qlMounter:
if err := s3qlMount(volumeID, ns.s3.cfg, targetPath); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("Error mounting bucket %s, invalid mounter specified: %s", volumeID, ns.s3.cfg.Mounter)
}
glog.V(4).Infof("s3: bucket %s successfuly mounted to %s", volumeID, targetPath)
@ -121,29 +131,16 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
// Check arguments
if len(req.GetVolumeId()) == 0 {
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
}
if len(req.GetStagingTargetPath()) == 0 {
return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
func (ns *nodeServer) NodeStageVolume(
ctx context.Context,
req *csi.NodeStageVolumeRequest) (
*csi.NodeStageVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}
return &csi.NodeStageVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
// Check arguments
if len(req.GetVolumeId()) == 0 {
return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request")
}
if len(req.GetStagingTargetPath()) == 0 {
return nil, status.Error(codes.InvalidArgument, "Target path missing in request")
}
return &csi.NodeUnstageVolumeResponse{}, nil
func (ns *nodeServer) NodeUnstageVolume(
ctx context.Context,
req *csi.NodeUnstageVolumeRequest) (
*csi.NodeUnstageVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}

4
pkg/s3/s3-client.go
View file

@ -17,10 +17,6 @@ type s3Client struct {
minio *minio.Client
}
type bucketMetadata struct {
CapacityBytes int64
}
func newS3Client(cfg *Config) (*s3Client, error) {
var client = &s3Client{}

1
pkg/s3/s3-driver_test.go
View file

@ -36,6 +36,7 @@ func TestDriver(t *testing.T) {
AccessKeyID: "FJDSJ",
SecretAccessKey: "DSG643HGDS",
Endpoint: "http://127.0.0.1:9000",
EncryptionKey: "IskEwCuEg6drywi",
}
driver, err := NewS3("test-node", endpoint, cfg)
if err != nil {

129
pkg/s3/s3ql.go Normal file
View file

@ -0,0 +1,129 @@
package s3
import (
"bytes"
"fmt"
"io"
"net/url"
"os"
"os/exec"
"path"
"strings"
"gopkg.in/ini.v1"
)
type s3fsConfig struct {
url string
bucketURL string
login string
password string
passphrase string
options []string
ssl bool
targetPath string
}
const (
s3qlCmdMkfs = "mkfs.s3ql"
s3qlCmdMount = "mount.s3ql"
)
func newS3ql(bucket string, targetPath string, cfg *Config) (*s3fsConfig, error) {
url, err := url.Parse(cfg.Endpoint)
if err != nil {
return nil, err
}
ssl := url.Scheme != "http"
if strings.Contains(url.Scheme, "http") {
url.Scheme = "s3c"
}
s3ql := &s3fsConfig{
url: url.String(),
login: cfg.AccessKeyID,
password: cfg.SecretAccessKey,
passphrase: cfg.EncryptionKey,
ssl: ssl,
targetPath: targetPath,
}
url.Path = path.Join(url.Path, bucket)
s3ql.bucketURL = url.String()
if !ssl {
s3ql.options = []string{"--backend-options", "no-ssl"}
}
return s3ql, s3ql.writeConfig()
}
func s3qlCreate(bucket string, cfg *Config) error {
s3ql, err := newS3ql(bucket, "unknown", cfg)
if err != nil {
return err
}
return s3ql.create()
}
func s3qlMount(bucket string, cfg *Config, targetPath string) error {
s3ql, err := newS3ql(bucket, targetPath, cfg)
if err != nil {
return err
}
return s3ql.mount()
}
func s3qlCmd(s3qlCmd string, args []string, stdin io.Reader) error {
cmd := exec.Command(s3qlCmd, args...)
if stdin != nil {
cmd.Stdin = stdin
}
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("Error running s3ql command: %s", out)
}
return nil
}
func (cfg *s3fsConfig) create() error {
// force creation to ignore existing data
args := []string{
cfg.bucketURL,
"--force",
}
p := fmt.Sprintf("%s\n%s\n", cfg.passphrase, cfg.passphrase)
reader := bytes.NewReader([]byte(p))
return s3qlCmd(s3qlCmdMkfs, append(args, cfg.options...), reader)
}
func (cfg *s3fsConfig) mount() error {
args := []string{
cfg.bucketURL,
cfg.targetPath,
"--allow-other",
}
return s3qlCmd(s3qlCmdMount, append(args, cfg.options...), nil)
}
func (cfg *s3fsConfig) writeConfig() error {
s3qlIni := ini.Empty()
section, err := s3qlIni.NewSection("s3ql")
if err != nil {
return err
}
section.NewKey("storage-url", cfg.url)
section.NewKey("backend-login", cfg.login)
section.NewKey("backend-password", cfg.password)
section.NewKey("fs-passphrase", cfg.passphrase)
authDir := os.Getenv("HOME") + "/.s3ql"
authFile := authDir + "/authinfo2"
os.Mkdir(authDir, 0700)
s3qlIni.SaveTo(authFile)
os.Chmod(authFile, 0600)
return nil
}

18
test/Dockerfile
View file

@ -1,8 +1,24 @@
FROM golang:stretch
LABEL maintainers="Cyrill Troxler <cyrilltroxler@gmail.com>"
LABEL description="s3 fuse csi plugin"
ARG S3QL_VERSION=release-2.28
RUN apt-get update && \
apt-get install -y \
s3fs wget python3 python3-setuptools \
python3-dev python3-pip python3-llfuse pkg-config \
sqlite3 libsqlite3-dev python3-apsw cython && \
rm -rf /var/lib/apt/lists/*
RUN pip3 install defusedxml dugong requests pycrypto
WORKDIR /usr/src
RUN wget -q https://github.com/s3ql/s3ql/archive/${S3QL_VERSION}.tar.gz
RUN tar -xzf ${S3QL_VERSION}.tar.gz
WORKDIR /usr/src/s3ql-${S3QL_VERSION}
RUN python3 setup.py build_cython build_ext --inplace
RUN python3 setup.py install
RUN apt-get update && apt-get install -y s3fs && rm -rf /var/lib/apt/lists/*
RUN go get -u github.com/minio/minio && go install github.com/minio/minio/cmd
CMD ["/go/src/github.com/ctrox/csi-s3-driver/test/test.sh"]