Add support for pmem-csi plugin and e2e test

Issue-ID: MULTICLOUD-1046
Change-Id: I1853e071a99702c5e6f7ba9ca819746576fd0aca
Signed-off-by: Chen, Tingjie <tingjie.chen@intel.com>

2 files changed, 776 insertions, 0 deletions

diff --git a/kud/deployment_infra/images/pmem-csi-direct.yaml b/kud/deployment_infra/images/pmem-csi-direct.yaml
new file mode 100644
index 00000000..7591a493
--- /dev/null
+++ b/kud/deployment_infra/images/pmem-csi-direct.yaml
@@ -0,0 +1,375 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-external-provisioner-cfg
+  namespace: default
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-external-provisioner-runner
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-csi-provisioner-role-cfg
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pmem-csi-external-provisioner-cfg
+subjects:
+- kind: ServiceAccount
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-csi-provisioner-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pmem-csi-external-provisioner-runner
+subjects:
+- kind: ServiceAccount
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-controller
+  namespace: default
+spec:
+  ports:
+  - port: 10000
+  selector:
+    app: pmem-csi-controller
+    pmem-csi.intel.com/deployment: direct-production
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-metrics
+  namespace: default
+spec:
+  ports:
+  - port: 10010
+  selector:
+    app: pmem-csi-controller
+    pmem-csi.intel.com/deployment: direct-production
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-controller
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pmem-csi-controller
+      pmem-csi.intel.com/deployment: direct-production
+  serviceName: pmem-csi-controller
+  template:
+    metadata:
+      labels:
+        app: pmem-csi-controller
+        pmem-csi.intel.com/deployment: direct-production
+        pmem-csi.intel.com/webhook: ignore
+    spec:
+      containers:
+      - command:
+        - /usr/local/bin/pmem-csi-driver
+        - -v=3
+        - -drivername=pmem-csi.intel.com
+        - -mode=controller
+        - -endpoint=unix:///csi/csi-controller.sock
+        - -registryEndpoint=tcp://0.0.0.0:10000
+        - -metricsListen=:10010
+        - -nodeid=$(KUBE_NODE_NAME)
+        - -caFile=/certs/ca.crt
+        - -certFile=/certs/tls.crt
+        - -keyFile=/certs/tls.key
+        env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-driver
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/termination-log
+        volumeMounts:
+        - mountPath: /certs/
+          name: registry-cert
+        - mountPath: /csi
+          name: plugin-socket-dir
+      - args:
+        - --timeout=5m
+        - --v=3
+        - --csi-address=/csi/csi-controller.sock
+        - --feature-gates=Topology=true
+        - --strict-topology=true
+        - --timeout=5m
+        - --strict-topology=true
+        image: quay.io/k8scsi/csi-provisioner:v1.2.1
+        imagePullPolicy: Always
+        name: external-provisioner
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-socket-dir
+      serviceAccount: pmem-csi-controller
+      volumes:
+      - emptyDir: null
+        name: plugin-socket-dir
+      - name: registry-cert
+        secret:
+          secretName: pmem-csi-registry-secrets
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi-node
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: pmem-csi-node
+      pmem-csi.intel.com/deployment: direct-production
+  template:
+    metadata:
+      labels:
+        app: pmem-csi-node
+        pmem-csi.intel.com/deployment: direct-production
+        pmem-csi.intel.com/webhook: ignore
+    spec:
+      containers:
+      - command:
+        - /usr/local/bin/pmem-csi-driver
+        - -deviceManager=ndctl
+        - -v=3
+        - -drivername=pmem-csi.intel.com
+        - -mode=node
+        - -endpoint=$(CSI_ENDPOINT)
+        - -nodeid=$(KUBE_NODE_NAME)
+        - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001
+        - -registryEndpoint=tcp://pmem-csi-controller:10000
+        - -caFile=/certs/ca.crt
+        - -certFile=/certs/tls.crt
+        - -keyFile=/certs/tls.key
+        - -statePath=/var/lib/pmem-csi.intel.com
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/pmem-csi.intel.com/csi.sock
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: KUBE_POD_IP
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.podIP
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-driver
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/termination-log
+        volumeMounts:
+        - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
+          mountPropagation: Bidirectional
+          name: mountpoint-dir
+        - mountPath: /var/lib/kubelet/pods
+          mountPropagation: Bidirectional
+          name: pods-dir
+        - mountPath: /certs/
+          name: registry-cert
+        - mountPath: /dev
+          name: dev-dir
+        - mountPath: /var/lib/pmem-csi.intel.com
+          mountPropagation: Bidirectional
+          name: pmem-state-dir
+        - mountPath: /sys
+          name: sys-dir
+      - args:
+        - -v=3
+        - --kubelet-registration-path=/var/lib/pmem-csi.intel.com/csi.sock
+        - --csi-address=/pmem-csi/csi.sock
+        image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
+        imagePullPolicy: Always
+        name: driver-registrar
+        volumeMounts:
+        - mountPath: /pmem-csi
+          name: pmem-state-dir
+        - mountPath: /registration
+          name: registration-dir
+      nodeSelector:
+        storage: pmem
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: DirectoryOrCreate
+        name: registration-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/kubernetes.io/csi
+          type: DirectoryOrCreate
+        name: mountpoint-dir
+      - hostPath:
+          path: /var/lib/kubelet/pods
+          type: DirectoryOrCreate
+        name: pods-dir
+      - name: registry-cert
+        secret:
+          secretName: pmem-csi-node-secrets
+      - hostPath:
+          path: /var/lib/pmem-csi.intel.com
+          type: DirectoryOrCreate
+        name: pmem-state-dir
+      - hostPath:
+          path: /dev
+          type: DirectoryOrCreate
+        name: dev-dir
+      - hostPath:
+          path: /sys
+          type: DirectoryOrCreate
+        name: sys-dir
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: direct-production
+  name: pmem-csi.intel.com
+spec:
+  attachRequired: false
+  podInfoOnMount: true
+  #volumeLifecycleModes:
+  #- Persistent
+  #- Ephemeral
diff --git a/kud/deployment_infra/images/pmem-csi-lvm.yaml b/kud/deployment_infra/images/pmem-csi-lvm.yaml
new file mode 100644
index 00000000..efe7c52e
--- /dev/null
+++ b/kud/deployment_infra/images/pmem-csi-lvm.yaml
@@ -0,0 +1,401 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-external-provisioner-cfg
+  namespace: default
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-external-provisioner-runner
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-csi-provisioner-role-cfg
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pmem-csi-external-provisioner-cfg
+subjects:
+- kind: ServiceAccount
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-csi-provisioner-role
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pmem-csi-external-provisioner-runner
+subjects:
+- kind: ServiceAccount
+  name: pmem-csi-controller
+  namespace: default
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-controller
+  namespace: default
+spec:
+  ports:
+  - port: 10000
+  selector:
+    app: pmem-csi-controller
+    pmem-csi.intel.com/deployment: lvm-production
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-metrics
+  namespace: default
+spec:
+  ports:
+  - port: 10010
+  selector:
+    app: pmem-csi-controller
+    pmem-csi.intel.com/deployment: lvm-production
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-controller
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pmem-csi-controller
+      pmem-csi.intel.com/deployment: lvm-production
+  serviceName: pmem-csi-controller
+  template:
+    metadata:
+      labels:
+        app: pmem-csi-controller
+        pmem-csi.intel.com/deployment: lvm-production
+        pmem-csi.intel.com/webhook: ignore
+    spec:
+      containers:
+      - command:
+        - /usr/local/bin/pmem-csi-driver
+        - -v=3
+        - -drivername=pmem-csi.intel.com
+        - -mode=controller
+        - -endpoint=unix:///csi/csi-controller.sock
+        - -registryEndpoint=tcp://0.0.0.0:10000
+        - -metricsListen=:10010
+        - -nodeid=$(KUBE_NODE_NAME)
+        - -caFile=/certs/ca.crt
+        - -certFile=/certs/tls.crt
+        - -keyFile=/certs/tls.key
+        env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-driver
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/termination-log
+        volumeMounts:
+        - mountPath: /certs/
+          name: registry-cert
+        - mountPath: /csi
+          name: plugin-socket-dir
+      - args:
+        - --timeout=5m
+        - --v=3
+        - --csi-address=/csi/csi-controller.sock
+        - --feature-gates=Topology=true
+        - --strict-topology=true
+        - --timeout=5m
+        - --strict-topology=true
+        image: quay.io/k8scsi/csi-provisioner:v1.2.1
+        imagePullPolicy: Always
+        name: external-provisioner
+        volumeMounts:
+        - mountPath: /csi
+          name: plugin-socket-dir
+      serviceAccount: pmem-csi-controller
+      volumes:
+      - emptyDir: null
+        name: plugin-socket-dir
+      - name: registry-cert
+        secret:
+          secretName: pmem-csi-registry-secrets
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi-node
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: pmem-csi-node
+      pmem-csi.intel.com/deployment: lvm-production
+  template:
+    metadata:
+      labels:
+        app: pmem-csi-node
+        pmem-csi.intel.com/deployment: lvm-production
+        pmem-csi.intel.com/webhook: ignore
+    spec:
+      containers:
+      - command:
+        - /usr/local/bin/pmem-csi-driver
+        - -deviceManager=lvm
+        - -v=3
+        - -drivername=pmem-csi.intel.com
+        - -mode=node
+        - -endpoint=$(CSI_ENDPOINT)
+        - -nodeid=$(KUBE_NODE_NAME)
+        - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001
+        - -registryEndpoint=tcp://pmem-csi-controller:10000
+        - -caFile=/certs/ca.crt
+        - -certFile=/certs/tls.crt
+        - -keyFile=/certs/tls.key
+        - -statePath=/var/lib/pmem-csi.intel.com
+        env:
+        - name: CSI_ENDPOINT
+          value: unix:///var/lib/pmem-csi.intel.com/csi.sock
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: KUBE_POD_IP
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: status.podIP
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-driver
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/termination-log
+        volumeMounts:
+        - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
+          mountPropagation: Bidirectional
+          name: mountpoint-dir
+        - mountPath: /var/lib/kubelet/pods
+          mountPropagation: Bidirectional
+          name: pods-dir
+        - mountPath: /certs/
+          name: registry-cert
+        - mountPath: /dev
+          name: dev-dir
+        - mountPath: /var/lib/pmem-csi.intel.com
+          mountPropagation: Bidirectional
+          name: pmem-state-dir
+      - args:
+        - -v=3
+        - --kubelet-registration-path=/var/lib/pmem-csi.intel.com/csi.sock
+        - --csi-address=/pmem-csi/csi.sock
+        image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0
+        imagePullPolicy: Always
+        name: driver-registrar
+        volumeMounts:
+        - mountPath: /pmem-csi
+          name: pmem-state-dir
+        - mountPath: /registration
+          name: registration-dir
+      initContainers:
+      - command:
+        - /usr/local/bin/pmem-ns-init
+        - -v=3
+        env:
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/pmem-ns-init-termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-ns-init
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/pmem-ns-init-termination-log
+        volumeMounts:
+        - mountPath: /sys
+          name: sys-dir
+      - command:
+        - /usr/local/bin/pmem-vgm
+        - -v=3
+        env:
+        - name: TERMINATION_LOG_PATH
+          value: /tmp/pmem-vgm-termination-log
+        image: intel/pmem-csi-driver:canary
+        imagePullPolicy: Always
+        name: pmem-vgm
+        securityContext:
+          privileged: true
+        terminationMessagePath: /tmp/pmem-vgm-termination-log
+      nodeSelector:
+        storage: pmem
+      volumes:
+      - hostPath:
+          path: /var/lib/kubelet/plugins_registry/
+          type: DirectoryOrCreate
+        name: registration-dir
+      - hostPath:
+          path: /var/lib/kubelet/plugins/kubernetes.io/csi
+          type: DirectoryOrCreate
+        name: mountpoint-dir
+      - hostPath:
+          path: /var/lib/kubelet/pods
+          type: DirectoryOrCreate
+        name: pods-dir
+      - name: registry-cert
+        secret:
+          secretName: pmem-csi-node-secrets
+      - hostPath:
+          path: /var/lib/pmem-csi.intel.com
+          type: DirectoryOrCreate
+        name: pmem-state-dir
+      - hostPath:
+          path: /dev
+          type: DirectoryOrCreate
+        name: dev-dir
+      - hostPath:
+          path: /sys
+          type: DirectoryOrCreate
+        name: sys-dir
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  labels:
+    pmem-csi.intel.com/deployment: lvm-production
+  name: pmem-csi.intel.com
+spec:
+  attachRequired: false
+  podInfoOnMount: true
+  #volumeLifecycleModes:
+  #- Persistent
+  #- Ephemeral