From 5895d2d8db29d863b6507a15f0881e63051acf5d Mon Sep 17 00:00:00 2001 From: Nikolay Mitrofanov Date: Sun, 15 Oct 2023 18:20:02 +0300 Subject: [PATCH] feature: Add configuretion to run setup/teardown command in helper container instead of run with script (cherry picked from commit a4c1f7460640dad4dca856140cb14005f529ca86) --- README.md | 14 ++- examples/distroless/Dockerfile.helper | 13 ++ examples/distroless/Dockerfile.provisioner | 21 ++++ examples/distroless/README.md | 2 + examples/distroless/build_and_test.sh | 38 ++++++ examples/distroless/config.json | 10 ++ examples/distroless/go.mod | 3 + examples/distroless/helperPod.yaml | 9 ++ examples/distroless/kind.yaml | 5 + examples/distroless/kustomization.yaml | 16 +++ examples/distroless/local-path-storage.yaml | 124 ++++++++++++++++++++ examples/distroless/main.go | 63 ++++++++++ examples/distroless/sts.yaml | 33 ++++++ provisioner.go | 75 ++++++++---- 14 files changed, 402 insertions(+), 24 deletions(-) create mode 100644 examples/distroless/Dockerfile.helper create mode 100644 examples/distroless/Dockerfile.provisioner create mode 100644 examples/distroless/README.md create mode 100755 examples/distroless/build_and_test.sh create mode 100644 examples/distroless/config.json create mode 100644 examples/distroless/go.mod create mode 100644 examples/distroless/helperPod.yaml create mode 100644 examples/distroless/kind.yaml create mode 100644 examples/distroless/kustomization.yaml create mode 100644 examples/distroless/local-path-storage.yaml create mode 100644 examples/distroless/main.go create mode 100644 examples/distroless/sts.yaml diff --git a/README.md b/README.md index 606b549f8..2b92bcc7e 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,9 @@ data: "node":"yasker-lp-dev3", "paths":[] } - ] + ], + "setupCommand": "/manager", + "teardownCommand": "/manager" } setup: |- #!/bin/sh @@ -200,6 +202,16 @@ In addition `volumeBindingMode: Immediate` can be used in StorageClass definiti Please note that `nodePathMap` and `sharedFileSystemPath` are mutually exclusive. If `sharedFileSystemPath` is used, then `nodePathMap` must be set to `[]`. +The `setupCommand` and `teardownCommand` allow you to specify the path to binary files in helperPod that will be called when creating or deleting pvc respectively. This can be useful if you need to use distroless images for security reasons. See the examples/distroless directory for an example. A binary file can take the following parameters: +| Parameter | Description | +| -------------------- | ----------- | +| -p | Volume directory that should be created or removed. | -m | -p | Volume directory that should be created or removed. | +| -m | The PersistentVolume mode (`Block` or `Filesystem`). | -m | The PersistentVolume mode (`Block` or `Filesystem`). | +| -s | Requested volume size in bytes. | -s | Requested volume size in bytes. | +| -a | Action type. Can be `create` or `delete` | -a | -a | Action type. + +The `setupCommand` and `teardownCommand` have higher priority than the `setup` and `teardown` scripts from the ConfigMap. + ##### Rules The configuration must obey following rules: 1. `config.json` must be a valid json file. diff --git a/examples/distroless/Dockerfile.helper b/examples/distroless/Dockerfile.helper new file mode 100644 index 000000000..16e7bac1d --- /dev/null +++ b/examples/distroless/Dockerfile.helper @@ -0,0 +1,13 @@ +FROM golang:1.17-alpine AS builder + +COPY main.go /main.go +COPY go.mod /go.mod + +RUN cd / && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-extldflags -static -s -w" -o /manager && \ + chmod 777 /manager + +FROM scratch + +COPY --from=builder /manager /manager + +ENTRYPOINT [ "/manager" ] \ No newline at end of file diff --git a/examples/distroless/Dockerfile.provisioner b/examples/distroless/Dockerfile.provisioner new file mode 100644 index 000000000..2b062a01f --- /dev/null +++ b/examples/distroless/Dockerfile.provisioner @@ -0,0 +1,21 @@ +FROM golang:1.17-alpine AS builder + +ARG GIT_REPO +ARG GIT_BRANCH + +RUN apk add --no-cache git + +ENV GIT_REPO=$GIT_REPO +ENV GIT_BRANCH=$GIT_BRANCH + +RUN mkdir /src && \ + git clone --depth 1 --branch "${GIT_BRANCH}" "${GIT_REPO}" /src && \ + cd /src && \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X main.VERSION=dev -extldflags -static -s -w" -o /local-path-provisioner && \ + chmod 777 /local-path-provisioner + +FROM scratch + +COPY --from=builder /local-path-provisioner /local-path-provisioner + +ENTRYPOINT [ "/local-path-provisioner" ] \ No newline at end of file diff --git a/examples/distroless/README.md b/examples/distroless/README.md new file mode 100644 index 000000000..05c438e3a --- /dev/null +++ b/examples/distroless/README.md @@ -0,0 +1,2 @@ +# Overview +this is an example to use distroless image for local path provisioner diff --git a/examples/distroless/build_and_test.sh b/examples/distroless/build_and_test.sh new file mode 100755 index 000000000..66acfdc3c --- /dev/null +++ b/examples/distroless/build_and_test.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e + +source=$1 +branch=$2 + +if [ -z "$source" ]; then + source="https://github.com/rancher/local-path-provisioner.git" +fi + +if [ -z "$branch" ]; then + branch="master" +fi + +docker build --build-arg="GIT_REPO=$source" --build-arg="GIT_BRANCH=$branch" -t lpp-distroless-provider:v0.0.1 -f Dockerfile.provisioner . + +docker build -t lpp-distroless-helper:v0.0.1 -f Dockerfile.helper . + +kind create cluster --config=kind.yaml --name test-lpp-distroless + +kind load docker-image --name test-lpp-distroless lpp-distroless-provider:v0.0.1 lpp-distroless-provider:v0.0.1 + +kind load docker-image --name test-lpp-distroless lpp-distroless-helper:v0.0.1 lpp-distroless-helper:v0.0.1 + +kubectl apply -k . + +echo "Waiting 30 seconds before deploy sts" + +sleep 30 + +kubectl create -f sts.yaml + +echo "Waiting 15 seconds before getting pv" + +sleep 15 + +kubectl get pv \ No newline at end of file diff --git a/examples/distroless/config.json b/examples/distroless/config.json new file mode 100644 index 000000000..ce02a17a6 --- /dev/null +++ b/examples/distroless/config.json @@ -0,0 +1,10 @@ +{ + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ], + "setupCommand": "/manager", + "teardownCommand": "/manager" +} \ No newline at end of file diff --git a/examples/distroless/go.mod b/examples/distroless/go.mod new file mode 100644 index 000000000..becd5b327 --- /dev/null +++ b/examples/distroless/go.mod @@ -0,0 +1,3 @@ +module manager + +go 1.17 \ No newline at end of file diff --git a/examples/distroless/helperPod.yaml b/examples/distroless/helperPod.yaml new file mode 100644 index 000000000..44a04df2c --- /dev/null +++ b/examples/distroless/helperPod.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: helper-pod +spec: + containers: + - name: helper-pod + image: lpp-distroless-helper:v0.0.1 + imagePullPolicy: IfNotPresent \ No newline at end of file diff --git a/examples/distroless/kind.yaml b/examples/distroless/kind.yaml new file mode 100644 index 000000000..0fe29e735 --- /dev/null +++ b/examples/distroless/kind.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker \ No newline at end of file diff --git a/examples/distroless/kustomization.yaml b/examples/distroless/kustomization.yaml new file mode 100644 index 000000000..ba5ebdde4 --- /dev/null +++ b/examples/distroless/kustomization.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- local-path-storage.yaml + +configMapGenerator: +- name: local-path-config + namespace: local-path-storage + behavior: merge + files: + - helperPod.yaml + - config.json + +generatorOptions: + disableNameSuffixHash: true diff --git a/examples/distroless/local-path-storage.yaml b/examples/distroless/local-path-storage.yaml new file mode 100644 index 000000000..bec276bc5 --- /dev/null +++ b/examples/distroless/local-path-storage.yaml @@ -0,0 +1,124 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: + - apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "update", "delete"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: + - kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: lpp-distroless-provider:v0.0.1 + imagePullPolicy: IfNotPresent + command: + - /local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete \ No newline at end of file diff --git a/examples/distroless/main.go b/examples/distroless/main.go new file mode 100644 index 000000000..6367bb5ea --- /dev/null +++ b/examples/distroless/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "flag" + "fmt" + "os" + "syscall" +) + +var ( + dirMode string + path string + sizeInBytes string + action string +) + +const ( + SETUP = "create" + TEARDOWN = "delete" +) + +func init() { + flag.StringVar(&path, "p", "", "Absolute path") + flag.StringVar(&sizeInBytes, "s", "", "Size in bytes") + flag.StringVar(&dirMode, "m", "", "Dir mode") + flag.StringVar(&action, "a", "", fmt.Sprintf("Action name. Can be '%s' or '%s'", SETUP, TEARDOWN)) +} + +func main() { + flag.Parse() + if action != SETUP && action != TEARDOWN { + fmt.Fprintf(os.Stderr, "Incorrect action: %s\n", action) + os.Exit(1) + } + + if path == "" { + fmt.Fprintf(os.Stderr, "Path is empty\n") + os.Exit(1) + } + + if path == "/" { + fmt.Fprintf(os.Stderr, "Path cannot be '/'\n") + os.Exit(1) + } + + if action == TEARDOWN { + err := os.RemoveAll(path) + + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot remove directory %s: %s\n", path, err) + os.Exit(1) + } + return + } + + syscall.Umask(0) + + err := os.MkdirAll(path, 0777) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot create directory %s: %s\n", path, err) + os.Exit(1) + } +} diff --git a/examples/distroless/sts.yaml b/examples/distroless/sts.yaml new file mode 100644 index 000000000..9e94d6570 --- /dev/null +++ b/examples/distroless/sts.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: web +spec: + serviceName: "nginx" + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: registry.k8s.io/nginx-slim:0.8 + ports: + - containerPort: 80 + name: web + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html + volumeClaimTemplates: + - metadata: + name: www + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: local-path + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/provisioner.go b/provisioner.go index 8f13467d5..20548ee8b 100644 --- a/provisioner.go +++ b/provisioner.go @@ -85,6 +85,8 @@ type ConfigData struct { NodePathMap []*NodePathMapData `json:"nodePathMap,omitempty"` CmdTimeoutSeconds int `json:"cmdTimeoutSeconds,omitempty"` SharedFileSystemPath string `json:"sharedFileSystemPath,omitempty"` + SetupCommand string `json:"setupCommand,omitempty"` + TeardownCommand string `json:"teardownCommand,omitempty"` } type NodePathMap struct { @@ -95,6 +97,8 @@ type Config struct { NodePathMap map[string]*NodePathMap CmdTimeoutSeconds int SharedFileSystemPath string + SetupCommand string + TeardownCommand string } func NewProvisioner(ctx context.Context, kubeClient *clientset.Clientset, @@ -291,7 +295,12 @@ func (p *LocalPathProvisioner) Provision(ctx context.Context, opts pvController. } storage := pvc.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] - provisionCmd := []string{"/bin/sh", "/script/setup"} + provisionCmd := make([]string, 0, 2) + if p.config.SetupCommand == "" { + provisionCmd = append(provisionCmd, "/bin/sh", "/script/setup") + } else { + provisionCmd = append(provisionCmd, p.config.SetupCommand) + } if err := p.createHelperPod(ActionTypeCreate, provisionCmd, volumeOptions{ Name: name, Path: path, @@ -380,7 +389,12 @@ func (p *LocalPathProvisioner) Delete(ctx context.Context, pv *v1.PersistentVolu logrus.Infof("Deleting volume %v at %v:%v", pv.Name, node, path) } storage := pv.Spec.Capacity[v1.ResourceName(v1.ResourceStorage)] - cleanupCmd := []string{"/bin/sh", "/script/teardown"} + cleanupCmd := make([]string, 0, 2) + if p.config.TeardownCommand == "" { + cleanupCmd = append(cleanupCmd, "/bin/sh", "/script/teardown") + } else { + cleanupCmd = append(cleanupCmd, p.config.TeardownCommand) + } if err := p.createHelperPod(ActionTypeDelete, cleanupCmd, volumeOptions{ Name: pv.Name, Path: path, @@ -488,36 +502,47 @@ func (p *LocalPathProvisioner) createHelperPod(action ActionType, cmd []string, }, }, }, + } + lpvTolerations := []v1.Toleration{ { - Name: helperScriptVolName, + Operator: v1.TolerationOpExists, + }, + } + helperPod := p.helperPod.DeepCopy() + + keyToPathItems := make([]v1.KeyToPath, 0, 2) + + if p.config.SetupCommand == "" { + keyToPathItems = append(keyToPathItems, v1.KeyToPath{ + Key: "setup", + Path: "setup", + }) + } + + if p.config.TeardownCommand == "" { + keyToPathItems = append(keyToPathItems, v1.KeyToPath{ + Key: "teardown", + Path: "teardown", + }) + } + + if len(keyToPathItems) > 0 { + lpvVolumes = append(lpvVolumes, v1.Volume{ + Name: "script", VolumeSource: v1.VolumeSource{ ConfigMap: &v1.ConfigMapVolumeSource{ LocalObjectReference: v1.LocalObjectReference{ Name: p.configMapName, }, - Items: []v1.KeyToPath{ - { - Key: "setup", - Path: "setup", - }, - { - Key: "teardown", - Path: "teardown", - }, - }, + Items: keyToPathItems, }, }, - }, - } - lpvTolerations := []v1.Toleration{ - { - Operator: v1.TolerationOpExists, - }, + }) + + scriptMount := addVolumeMount(&helperPod.Spec.Containers[0].VolumeMounts, helperScriptVolName, helperScriptDir) + scriptMount.MountPath = helperScriptDir } - helperPod := p.helperPod.DeepCopy() - scriptMount := addVolumeMount(&helperPod.Spec.Containers[0].VolumeMounts, helperScriptVolName, helperScriptDir) - scriptMount.MountPath = helperScriptDir dataMount := addVolumeMount(&helperPod.Spec.Containers[0].VolumeMounts, helperDataVolName, parentDir) parentDir = dataMount.MountPath parentDir = strings.TrimSuffix(parentDir, string(filepath.Separator)) @@ -551,7 +576,8 @@ func (p *LocalPathProvisioner) createHelperPod(action ActionType, cmd []string, helperPod.Spec.Containers[0].Env = append(helperPod.Spec.Containers[0].Env, env...) helperPod.Spec.Containers[0].Args = []string{"-p", filepath.Join(parentDir, volumeDir), "-s", strconv.FormatInt(o.SizeInBytes, 10), - "-m", string(o.Mode)} + "-m", string(o.Mode), + "-a", string(action)} helperPod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ Privileged: &privileged, } @@ -650,6 +676,8 @@ func canonicalizeConfig(data *ConfigData) (cfg *Config, err error) { }() cfg = &Config{} cfg.SharedFileSystemPath = data.SharedFileSystemPath + cfg.SetupCommand = data.SetupCommand + cfg.TeardownCommand = data.TeardownCommand cfg.NodePathMap = map[string]*NodePathMap{} for _, n := range data.NodePathMap { if cfg.NodePathMap[n.Node] != nil { @@ -679,6 +707,7 @@ func canonicalizeConfig(data *ConfigData) (cfg *Config, err error) { } else { cfg.CmdTimeoutSeconds = defaultCmdTimeoutSeconds } + return cfg, nil }