From 22f755508a107c689f325042c4abaa98c5bd450e Mon Sep 17 00:00:00 2001 From: "Chen, Tingjie" Date: Mon, 25 May 2020 14:26:26 +0800 Subject: Add support for pmem-csi plugin and e2e test Issue-ID: MULTICLOUD-1046 Change-Id: I1853e071a99702c5e6f7ba9ca819746576fd0aca Signed-off-by: Chen, Tingjie --- kud/deployment_infra/images/pmem-csi-direct.yaml | 375 +++++++++++++++++++++ kud/deployment_infra/images/pmem-csi-lvm.yaml | 401 +++++++++++++++++++++++ 2 files changed, 776 insertions(+) create mode 100644 kud/deployment_infra/images/pmem-csi-direct.yaml create mode 100644 kud/deployment_infra/images/pmem-csi-lvm.yaml (limited to 'kud/deployment_infra/images') diff --git a/kud/deployment_infra/images/pmem-csi-direct.yaml b/kud/deployment_infra/images/pmem-csi-direct.yaml new file mode 100644 index 00000000..7591a493 --- /dev/null +++ b/kud/deployment_infra/images/pmem-csi-direct.yaml @@ -0,0 +1,375 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-controller + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-external-provisioner-cfg + namespace: default +rules: +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-external-provisioner-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-csi-provisioner-role-cfg + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-controller + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-external-provisioner-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-controller + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-controller + namespace: default +spec: + ports: + - port: 10000 + selector: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: direct-production +--- +apiVersion: v1 +kind: Service +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-metrics + namespace: default +spec: + ports: + - port: 10010 + selector: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: direct-production + type: NodePort +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-controller + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: direct-production + serviceName: pmem-csi-controller + template: + metadata: + labels: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: direct-production + pmem-csi.intel.com/webhook: ignore + spec: + containers: + - command: + - /usr/local/bin/pmem-csi-driver + - -v=3 + - -drivername=pmem-csi.intel.com + - -mode=controller + - -endpoint=unix:///csi/csi-controller.sock + - -registryEndpoint=tcp://0.0.0.0:10000 + - -metricsListen=:10010 + - -nodeid=$(KUBE_NODE_NAME) + - -caFile=/certs/ca.crt + - -certFile=/certs/tls.crt + - -keyFile=/certs/tls.key + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: TERMINATION_LOG_PATH + value: /tmp/termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-driver + securityContext: + privileged: true + terminationMessagePath: /tmp/termination-log + volumeMounts: + - mountPath: /certs/ + name: registry-cert + - mountPath: /csi + name: plugin-socket-dir + - args: + - --timeout=5m + - --v=3 + - --csi-address=/csi/csi-controller.sock + - --feature-gates=Topology=true + - --strict-topology=true + - --timeout=5m + - --strict-topology=true + image: quay.io/k8scsi/csi-provisioner:v1.2.1 + imagePullPolicy: Always + name: external-provisioner + volumeMounts: + - mountPath: /csi + name: plugin-socket-dir + serviceAccount: pmem-csi-controller + volumes: + - emptyDir: null + name: plugin-socket-dir + - name: registry-cert + secret: + secretName: pmem-csi-registry-secrets +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi-node + namespace: default +spec: + selector: + matchLabels: + app: pmem-csi-node + pmem-csi.intel.com/deployment: direct-production + template: + metadata: + labels: + app: pmem-csi-node + pmem-csi.intel.com/deployment: direct-production + pmem-csi.intel.com/webhook: ignore + spec: + containers: + - command: + - /usr/local/bin/pmem-csi-driver + - -deviceManager=ndctl + - -v=3 + - -drivername=pmem-csi.intel.com + - -mode=node + - -endpoint=$(CSI_ENDPOINT) + - -nodeid=$(KUBE_NODE_NAME) + - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 + - -registryEndpoint=tcp://pmem-csi-controller:10000 + - -caFile=/certs/ca.crt + - -certFile=/certs/tls.crt + - -keyFile=/certs/tls.key + - -statePath=/var/lib/pmem-csi.intel.com + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/pmem-csi.intel.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: KUBE_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: TERMINATION_LOG_PATH + value: /tmp/termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-driver + securityContext: + privileged: true + terminationMessagePath: /tmp/termination-log + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pods-dir + - mountPath: /certs/ + name: registry-cert + - mountPath: /dev + name: dev-dir + - mountPath: /var/lib/pmem-csi.intel.com + mountPropagation: Bidirectional + name: pmem-state-dir + - mountPath: /sys + name: sys-dir + - args: + - -v=3 + - --kubelet-registration-path=/var/lib/pmem-csi.intel.com/csi.sock + - --csi-address=/pmem-csi/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + imagePullPolicy: Always + name: driver-registrar + volumeMounts: + - mountPath: /pmem-csi + name: pmem-state-dir + - mountPath: /registration + name: registration-dir + nodeSelector: + storage: pmem + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: pods-dir + - name: registry-cert + secret: + secretName: pmem-csi-node-secrets + - hostPath: + path: /var/lib/pmem-csi.intel.com + type: DirectoryOrCreate + name: pmem-state-dir + - hostPath: + path: /dev + type: DirectoryOrCreate + name: dev-dir + - hostPath: + path: /sys + type: DirectoryOrCreate + name: sys-dir +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + labels: + pmem-csi.intel.com/deployment: direct-production + name: pmem-csi.intel.com +spec: + attachRequired: false + podInfoOnMount: true + #volumeLifecycleModes: + #- Persistent + #- Ephemeral diff --git a/kud/deployment_infra/images/pmem-csi-lvm.yaml b/kud/deployment_infra/images/pmem-csi-lvm.yaml new file mode 100644 index 00000000..efe7c52e --- /dev/null +++ b/kud/deployment_infra/images/pmem-csi-lvm.yaml @@ -0,0 +1,401 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-controller + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-external-provisioner-cfg + namespace: default +rules: +- apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - watch + - list + - delete + - update + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-external-provisioner-runner +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-csi-provisioner-role-cfg + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pmem-csi-external-provisioner-cfg +subjects: +- kind: ServiceAccount + name: pmem-csi-controller + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-csi-provisioner-role +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pmem-csi-external-provisioner-runner +subjects: +- kind: ServiceAccount + name: pmem-csi-controller + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-controller + namespace: default +spec: + ports: + - port: 10000 + selector: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: lvm-production +--- +apiVersion: v1 +kind: Service +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-metrics + namespace: default +spec: + ports: + - port: 10010 + selector: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: lvm-production + type: NodePort +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-controller + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: lvm-production + serviceName: pmem-csi-controller + template: + metadata: + labels: + app: pmem-csi-controller + pmem-csi.intel.com/deployment: lvm-production + pmem-csi.intel.com/webhook: ignore + spec: + containers: + - command: + - /usr/local/bin/pmem-csi-driver + - -v=3 + - -drivername=pmem-csi.intel.com + - -mode=controller + - -endpoint=unix:///csi/csi-controller.sock + - -registryEndpoint=tcp://0.0.0.0:10000 + - -metricsListen=:10010 + - -nodeid=$(KUBE_NODE_NAME) + - -caFile=/certs/ca.crt + - -certFile=/certs/tls.crt + - -keyFile=/certs/tls.key + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: TERMINATION_LOG_PATH + value: /tmp/termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-driver + securityContext: + privileged: true + terminationMessagePath: /tmp/termination-log + volumeMounts: + - mountPath: /certs/ + name: registry-cert + - mountPath: /csi + name: plugin-socket-dir + - args: + - --timeout=5m + - --v=3 + - --csi-address=/csi/csi-controller.sock + - --feature-gates=Topology=true + - --strict-topology=true + - --timeout=5m + - --strict-topology=true + image: quay.io/k8scsi/csi-provisioner:v1.2.1 + imagePullPolicy: Always + name: external-provisioner + volumeMounts: + - mountPath: /csi + name: plugin-socket-dir + serviceAccount: pmem-csi-controller + volumes: + - emptyDir: null + name: plugin-socket-dir + - name: registry-cert + secret: + secretName: pmem-csi-registry-secrets +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi-node + namespace: default +spec: + selector: + matchLabels: + app: pmem-csi-node + pmem-csi.intel.com/deployment: lvm-production + template: + metadata: + labels: + app: pmem-csi-node + pmem-csi.intel.com/deployment: lvm-production + pmem-csi.intel.com/webhook: ignore + spec: + containers: + - command: + - /usr/local/bin/pmem-csi-driver + - -deviceManager=lvm + - -v=3 + - -drivername=pmem-csi.intel.com + - -mode=node + - -endpoint=$(CSI_ENDPOINT) + - -nodeid=$(KUBE_NODE_NAME) + - -controllerEndpoint=tcp://$(KUBE_POD_IP):10001 + - -registryEndpoint=tcp://pmem-csi-controller:10000 + - -caFile=/certs/ca.crt + - -certFile=/certs/tls.crt + - -keyFile=/certs/tls.key + - -statePath=/var/lib/pmem-csi.intel.com + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/pmem-csi.intel.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: KUBE_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: TERMINATION_LOG_PATH + value: /tmp/termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-driver + securityContext: + privileged: true + terminationMessagePath: /tmp/termination-log + volumeMounts: + - mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: pods-dir + - mountPath: /certs/ + name: registry-cert + - mountPath: /dev + name: dev-dir + - mountPath: /var/lib/pmem-csi.intel.com + mountPropagation: Bidirectional + name: pmem-state-dir + - args: + - -v=3 + - --kubelet-registration-path=/var/lib/pmem-csi.intel.com/csi.sock + - --csi-address=/pmem-csi/csi.sock + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + imagePullPolicy: Always + name: driver-registrar + volumeMounts: + - mountPath: /pmem-csi + name: pmem-state-dir + - mountPath: /registration + name: registration-dir + initContainers: + - command: + - /usr/local/bin/pmem-ns-init + - -v=3 + env: + - name: TERMINATION_LOG_PATH + value: /tmp/pmem-ns-init-termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-ns-init + securityContext: + privileged: true + terminationMessagePath: /tmp/pmem-ns-init-termination-log + volumeMounts: + - mountPath: /sys + name: sys-dir + - command: + - /usr/local/bin/pmem-vgm + - -v=3 + env: + - name: TERMINATION_LOG_PATH + value: /tmp/pmem-vgm-termination-log + image: intel/pmem-csi-driver:canary + imagePullPolicy: Always + name: pmem-vgm + securityContext: + privileged: true + terminationMessagePath: /tmp/pmem-vgm-termination-log + nodeSelector: + storage: pmem + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins/kubernetes.io/csi + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: pods-dir + - name: registry-cert + secret: + secretName: pmem-csi-node-secrets + - hostPath: + path: /var/lib/pmem-csi.intel.com + type: DirectoryOrCreate + name: pmem-state-dir + - hostPath: + path: /dev + type: DirectoryOrCreate + name: dev-dir + - hostPath: + path: /sys + type: DirectoryOrCreate + name: sys-dir +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + labels: + pmem-csi.intel.com/deployment: lvm-production + name: pmem-csi.intel.com +spec: + attachRequired: false + podInfoOnMount: true + #volumeLifecycleModes: + #- Persistent + #- Ephemeral -- cgit 1.2.3-korg