Add support for pmem-csi plugin and e2e test

Issue-ID: MULTICLOUD-1046
Change-Id: I1853e071a99702c5e6f7ba9ca819746576fd0aca
Signed-off-by: Chen, Tingjie <tingjie.chen@intel.com>

7 files changed, 298 insertions, 0 deletions

diff --git a/kud/deployment_infra/playbooks/configure-optane.yml b/kud/deployment_infra/playbooks/configure-optane.yml
new file mode 100644
index 00000000..8e000aa4
--- /dev/null
+++ b/kud/deployment_infra/playbooks/configure-optane.yml
@@ -0,0 +1,15 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- import_playbook: preconfigure-optane.yml
+- hosts: localhost
+  tasks:
+    - name: Apply Optane PMEM CSI Daemonset
+      command: "{{ base_dest }}/optane/deploy_optane.sh"
diff --git a/kud/deployment_infra/playbooks/deploy_optane.sh b/kud/deployment_infra/playbooks/deploy_optane.sh
new file mode 100755
index 00000000..cb502373
--- /dev/null
+++ b/kud/deployment_infra/playbooks/deploy_optane.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+work_path="$(dirname -- "$(readlink -f -- "$0")")"
+ndctl_region=`ndctl list -R`
+if [[ $ndctl_region == "" ]] ; then
+    echo "No Optane Hardware!"
+else
+    echo "Optane Plugin start .."
+    /usr/local/bin/kubectl apply -f $work_path/pmem-csi-lvm.yaml
+fi
diff --git a/kud/deployment_infra/playbooks/install_optane.sh b/kud/deployment_infra/playbooks/install_optane.sh
new file mode 100755
index 00000000..3f345362
--- /dev/null
+++ b/kud/deployment_infra/playbooks/install_optane.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Precondition:
+# Optane PM related utilities download and configure.
+
+# collet and install ndctl and check hardware
+echo "[OPTANE] Install ndctl ..."
+apt install -y ndctl
+
+echo "[OPTANE] Check the NVDIMM hardware ..."
+ndctl_region=`ndctl list -R`
+if [[ $ndctl_region == "" ]] ; then
+    echo "No NVDIMM hardware, exit ..."
+    exit 0
+fi
+
+# get current folder path
+work_path="$(dirname -- "$(readlink -f -- "$0")")"
+node_name="$(kubectl get node -o jsonpath='{.items[0].metadata.name}')"
+
+# collet and install ipmctl
+echo "[OPTANE] Install ipmctl ..."
+cd $work_path
+wget https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/02.00.00.3474+really01.00.00.3469-1/ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz
+tar xvf ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz
+cd ipmctl-01.00.00.3469/
+
+echo "[OPTANE] Install ipmctl utilities"
+mkdir output && cd output
+apt install -y cmake build-essential pkg-config asciidoctor asciidoc libndctl-dev git
+gem install asciidoctor-pdf --pre
+
+add-apt-repository --yes ppa:jhli/libsafec
+apt update
+apt-get install -y libsafec-dev
+
+echo "[OPTANE] Build ipmctl ..."
+cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ ..
+make -j all
+make install
+
+cd $work_path
+
+echo "[OPTANE] Install cfssl tools ..."
+# collect cfssl tools
+curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o cfssl
+curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o cfssljson
+chmod a+x cfssl cfssljson
+cp -rf cfssl cfssljson /usr/bin/
+
+echo "[OPTANE] Create AppDirect Goal ..."
+# ipmctl setting
+#ipmctl delete -goal
+#ipmctl create -f -goal PersistentMemoryType=AppDirectNotInterleaved
+
+# Run certificates set-up script
+echo "[OPTANE] Run ca for kubernetes ..."
+./setup-ca-kubernetes.sh
+
+# deploy docker hub
+echo "[OPTANE] Set label node for storage pmem ..."
+kubectl label node $node_name storage=pmem
+
+echo "[OPTANE] kubelet CSIMigration set false ..."
+echo -e "featureGates:\n  CSIMigration: false" >> /var/lib/kubelet/config.yaml
+# deploy pmem-csi and applications
+# select two mode: lvm and direct
+#echo "[OPTANE] Create PMEM-CSI plugin service ..."
+#kubectl create -f ../images/pmem-csi-lvm.yaml
+# kubectl create -f pmem-csi-direct.yaml
+
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 7a25670d..e5438ae4 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -73,3 +73,9 @@ qat_driver_source_type: "tarball"
 qat_driver_version: 1.7.l.4.6.0-00025
 qat_driver_url: "https://01.org/sites/default/files/downloads/{{ qat_package }}.tar.gz"
 qat_package: qat1.7.l.4.6.0-00025
+
+optane_dest: "{{ base_dest }}/optane"
+optane_ipmctl_source_type: "tarball"
+optane_ipmctl_version: 02.00.00.3474
+optane_ipmctl_url: "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/{{ optane_package }}.tar.xz"
+optane_ipmctl_package: ipmctl_02.00.00.3474+really01.00.00.3469.orig
diff --git a/kud/deployment_infra/playbooks/preconfigure-optane.yml b/kud/deployment_infra/playbooks/preconfigure-optane.yml
new file mode 100644
index 00000000..64622895
--- /dev/null
+++ b/kud/deployment_infra/playbooks/preconfigure-optane.yml
@@ -0,0 +1,85 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Run the following task only if the SRIOV is set to True
+# i.e when SRIOV hardware is available
+- hosts: localhost
+  become: yes
+  pre_tasks:
+    - name: Load kud variables
+      include_vars:
+        file: kud-vars.yml
+  tasks:
+    - name: Create optane folder
+      file:
+        state: directory
+        path: "{{ optane_dest }}"
+      ignore_errors: yes
+
+- hosts: kube-node
+  become: yes
+  pre_tasks:
+    - name: Load kud variables
+      include_vars:
+        file: kud-vars.yml
+  tasks:
+    - name: Create OPTANE folder in the target destination
+      file:
+        state: directory
+        path: "{{ item }}"
+      with_items:
+        - optane
+    - copy:
+        src: "{{ playbook_dir }}/setup-ca-kubernetes.sh"
+        dest: optane
+    - name: Changing perm of "sh", adding "+x"
+      shell: "chmod +x setup-ca-kubernetes.sh"
+      args:
+        chdir: "optane"
+        warn: False
+    - copy:
+        src: "{{ playbook_dir }}/setup-ca.sh"
+        dest: optane
+    - name: Changing perm of "sh", adding "+x"
+      shell: "chmod +x setup-ca.sh"
+      args:
+        chdir: "optane"
+        warn: False
+    - copy:
+        src: "{{ playbook_dir }}/install_optane.sh"
+        dest: optane
+    - name: Changing perm of "sh", adding "+x"
+      shell: "chmod +x install_optane.sh"
+      args:
+        chdir: "optane"
+        warn: False
+    - copy:
+        src: "{{ playbook_dir }}/deploy_optane.sh"
+        dest: optane
+    - name: Changing perm of "sh", adding "+x"
+      shell: "chmod +x deploy_optane.sh"
+      args:
+        chdir: "optane"
+        warn: False
+    - copy:
+        src: "{{ playbook_dir }}/../images/pmem-csi-lvm.yaml"
+        dest: optane
+    - copy:
+        src: "{{ playbook_dir }}/../images/pmem-csi-direct.yaml"
+        dest: optane
+    - name: Run the install script and Re-evaluate the variable
+      command: optane/install_optane.sh
+      register: output
+    - name: restart kubelet services
+      become: yes
+      service:
+        name: kubelet
+        state: restarted
+
diff --git a/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh
new file mode 100755
index 00000000..c436f1ca
--- /dev/null
+++ b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+
+# This script generates certificates using setup-ca.sh and converts them into
+# the Kubernetes secrets that the PMEM-CSI deployments rely upon for
+# securing communication between PMEM-CSI components. Existing secrets
+# are updated with new certificates when running it again.
+
+# The script needs a functional kubectl that uses the target cluster.
+: ${KUBECTL:=kubectl}
+
+# The directory containing setup-ca*.sh.
+: ${TEST_DIRECTORY:=$(dirname $(readlink -f $0))}
+
+
+tmpdir=`mktemp -d`
+trap 'rm -r $tmpdir' EXIT
+
+# Generate certificates. They are not going to be needed again and will
+# be deleted together with the temp directory.
+WORKDIR="$tmpdir" "$TEST_DIRECTORY/setup-ca.sh"
+
+# This reads a file and encodes it for use in a secret.
+read_key () {
+    base64 -w 0 "$1"
+}
+
+# Read certificate files and turn them into Kubernetes secrets.
+#
+# -caFile (controller and all nodes)
+CA=$(read_key "$tmpdir/ca.pem")
+# -certFile (controller)
+REGISTRY_CERT=$(read_key "$tmpdir/pmem-registry.pem")
+# -keyFile (controller)
+REGISTRY_KEY=$(read_key "$tmpdir/pmem-registry-key.pem")
+# -certFile (same for all nodes)
+NODE_CERT=$(read_key "$tmpdir/pmem-node-controller.pem")
+# -keyFile (same for all nodes)
+NODE_KEY=$(read_key "$tmpdir/pmem-node-controller-key.pem")
+
+${KUBECTL} apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+    name: pmem-csi-registry-secrets
+type: kubernetes.io/tls
+data:
+    ca.crt: ${CA}
+    tls.crt: ${REGISTRY_CERT}
+    tls.key: ${REGISTRY_KEY}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: pmem-csi-node-secrets
+type: Opaque
+data:
+    ca.crt: ${CA}
+    tls.crt: ${NODE_CERT}
+    tls.key: ${NODE_KEY}
+EOF
diff --git a/kud/deployment_infra/playbooks/setup-ca.sh b/kud/deployment_infra/playbooks/setup-ca.sh
new file mode 100755
index 00000000..77addc71
--- /dev/null
+++ b/kud/deployment_infra/playbooks/setup-ca.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# Directory to use for storing intermediate files.
+CA=${CA:="pmem-ca"}
+WORKDIR=${WORKDIR:-$(mktemp -d -u -t pmem-XXXX)}
+mkdir -p $WORKDIR
+cd $WORKDIR
+
+# Check for cfssl utilities.
+cfssl_found=1
+(command -v cfssl 2>&1 >/dev/null && command -v cfssljson 2>&1 >/dev/null) || cfssl_found=0
+if [ $cfssl_found -eq 0 ]; then
+    echo "cfssl tools not found, Please install cfssl and cfssljson."
+    exit 1
+fi
+
+# Generate CA certificates.
+<<EOF cfssl -loglevel=3 gencert -initca - | cfssljson -bare ca
+{
+    "CN": "$CA",
+    "key": {
+        "algo": "rsa",
+        "size": 2048
+    }
+}
+EOF
+
+# Generate server and client certificates.
+DEFAULT_CNS="pmem-registry pmem-node-controller"
+CNS="${DEFAULT_CNS} ${EXTRA_CNS:=""}"
+for name in ${CNS}; do
+  <<EOF cfssl -loglevel=3 gencert -ca=ca.pem -ca-key=ca-key.pem - | cfssljson -bare $name
+{
+    "CN": "$name",
+    "hosts": [
+        $(if [ "$name" = "pmem-registry" ]; then
+             # Some extra names needed for scheduler extender and webhook.
+             echo '"pmem-csi-scheduler", "pmem-csi-scheduler.default", "pmem-csi-scheduler.default.svc", "127.0.0.1",'
+             # And for metrics server.
+             echo '"pmem-csi-metrics", "pmem-csi-metrics.default", "pmem-csi-metrics.default.svc",'
+          fi
+        )
+        "$name"
+    ],
+    "key": {
+        "algo": "ecdsa",
+        "size": 256
+    }
+}
+EOF
+done