aboutsummaryrefslogtreecommitdiffstats
path: root/kud/deployment_infra/playbooks
diff options
context:
space:
mode:
authorChen, Tingjie <tingjie.chen@intel.com>2020-05-25 14:26:26 +0800
committerChen, Tingjie <tingjie.chen@intel.com>2020-05-30 11:48:31 +0800
commit22f755508a107c689f325042c4abaa98c5bd450e (patch)
tree669bf0079f12b96f156d7cfc7041f7ee5c8b9f2f /kud/deployment_infra/playbooks
parent079d214800484b0bfedd22fc72f4b3c61ecc6f83 (diff)
Add support for pmem-csi plugin and e2e test
Issue-ID: MULTICLOUD-1046 Change-Id: I1853e071a99702c5e6f7ba9ca819746576fd0aca Signed-off-by: Chen, Tingjie <tingjie.chen@intel.com>
Diffstat (limited to 'kud/deployment_infra/playbooks')
-rw-r--r--kud/deployment_infra/playbooks/configure-optane.yml15
-rwxr-xr-xkud/deployment_infra/playbooks/deploy_optane.sh10
-rwxr-xr-xkud/deployment_infra/playbooks/install_optane.sh71
-rw-r--r--kud/deployment_infra/playbooks/kud-vars.yml6
-rw-r--r--kud/deployment_infra/playbooks/preconfigure-optane.yml85
-rwxr-xr-xkud/deployment_infra/playbooks/setup-ca-kubernetes.sh60
-rwxr-xr-xkud/deployment_infra/playbooks/setup-ca.sh51
7 files changed, 298 insertions, 0 deletions
diff --git a/kud/deployment_infra/playbooks/configure-optane.yml b/kud/deployment_infra/playbooks/configure-optane.yml
new file mode 100644
index 00000000..8e000aa4
--- /dev/null
+++ b/kud/deployment_infra/playbooks/configure-optane.yml
@@ -0,0 +1,15 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- import_playbook: preconfigure-optane.yml
+- hosts: localhost
+ tasks:
+ - name: Apply Optane PMEM CSI Daemonset
+ command: "{{ base_dest }}/optane/deploy_optane.sh"
diff --git a/kud/deployment_infra/playbooks/deploy_optane.sh b/kud/deployment_infra/playbooks/deploy_optane.sh
new file mode 100755
index 00000000..cb502373
--- /dev/null
+++ b/kud/deployment_infra/playbooks/deploy_optane.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+work_path="$(dirname -- "$(readlink -f -- "$0")")"
+ndctl_region=`ndctl list -R`
+if [[ $ndctl_region == "" ]] ; then
+ echo "No Optane Hardware!"
+else
+ echo "Optane Plugin start .."
+ /usr/local/bin/kubectl apply -f $work_path/pmem-csi-lvm.yaml
+fi
diff --git a/kud/deployment_infra/playbooks/install_optane.sh b/kud/deployment_infra/playbooks/install_optane.sh
new file mode 100755
index 00000000..3f345362
--- /dev/null
+++ b/kud/deployment_infra/playbooks/install_optane.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Precondition:
+# Optane PM related utilities download and configure.
+
+# collet and install ndctl and check hardware
+echo "[OPTANE] Install ndctl ..."
+apt install -y ndctl
+
+echo "[OPTANE] Check the NVDIMM hardware ..."
+ndctl_region=`ndctl list -R`
+if [[ $ndctl_region == "" ]] ; then
+ echo "No NVDIMM hardware, exit ..."
+ exit 0
+fi
+
+# get current folder path
+work_path="$(dirname -- "$(readlink -f -- "$0")")"
+node_name="$(kubectl get node -o jsonpath='{.items[0].metadata.name}')"
+
+# collet and install ipmctl
+echo "[OPTANE] Install ipmctl ..."
+cd $work_path
+wget https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/02.00.00.3474+really01.00.00.3469-1/ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz
+tar xvf ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz
+cd ipmctl-01.00.00.3469/
+
+echo "[OPTANE] Install ipmctl utilities"
+mkdir output && cd output
+apt install -y cmake build-essential pkg-config asciidoctor asciidoc libndctl-dev git
+gem install asciidoctor-pdf --pre
+
+add-apt-repository --yes ppa:jhli/libsafec
+apt update
+apt-get install -y libsafec-dev
+
+echo "[OPTANE] Build ipmctl ..."
+cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ ..
+make -j all
+make install
+
+cd $work_path
+
+echo "[OPTANE] Install cfssl tools ..."
+# collect cfssl tools
+curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o cfssl
+curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o cfssljson
+chmod a+x cfssl cfssljson
+cp -rf cfssl cfssljson /usr/bin/
+
+echo "[OPTANE] Create AppDirect Goal ..."
+# ipmctl setting
+#ipmctl delete -goal
+#ipmctl create -f -goal PersistentMemoryType=AppDirectNotInterleaved
+
+# Run certificates set-up script
+echo "[OPTANE] Run ca for kubernetes ..."
+./setup-ca-kubernetes.sh
+
+# deploy docker hub
+echo "[OPTANE] Set label node for storage pmem ..."
+kubectl label node $node_name storage=pmem
+
+echo "[OPTANE] kubelet CSIMigration set false ..."
+echo -e "featureGates:\n CSIMigration: false" >> /var/lib/kubelet/config.yaml
+# deploy pmem-csi and applications
+# select two mode: lvm and direct
+#echo "[OPTANE] Create PMEM-CSI plugin service ..."
+#kubectl create -f ../images/pmem-csi-lvm.yaml
+# kubectl create -f pmem-csi-direct.yaml
+
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 7a25670d..e5438ae4 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -73,3 +73,9 @@ qat_driver_source_type: "tarball"
qat_driver_version: 1.7.l.4.6.0-00025
qat_driver_url: "https://01.org/sites/default/files/downloads/{{ qat_package }}.tar.gz"
qat_package: qat1.7.l.4.6.0-00025
+
+optane_dest: "{{ base_dest }}/optane"
+optane_ipmctl_source_type: "tarball"
+optane_ipmctl_version: 02.00.00.3474
+optane_ipmctl_url: "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/{{ optane_package }}.tar.xz"
+optane_ipmctl_package: ipmctl_02.00.00.3474+really01.00.00.3469.orig
diff --git a/kud/deployment_infra/playbooks/preconfigure-optane.yml b/kud/deployment_infra/playbooks/preconfigure-optane.yml
new file mode 100644
index 00000000..64622895
--- /dev/null
+++ b/kud/deployment_infra/playbooks/preconfigure-optane.yml
@@ -0,0 +1,85 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Run the following task only if the SRIOV is set to True
+# i.e when SRIOV hardware is available
+- hosts: localhost
+ become: yes
+ pre_tasks:
+ - name: Load kud variables
+ include_vars:
+ file: kud-vars.yml
+ tasks:
+ - name: Create optane folder
+ file:
+ state: directory
+ path: "{{ optane_dest }}"
+ ignore_errors: yes
+
+- hosts: kube-node
+ become: yes
+ pre_tasks:
+ - name: Load kud variables
+ include_vars:
+ file: kud-vars.yml
+ tasks:
+ - name: Create OPTANE folder in the target destination
+ file:
+ state: directory
+ path: "{{ item }}"
+ with_items:
+ - optane
+ - copy:
+ src: "{{ playbook_dir }}/setup-ca-kubernetes.sh"
+ dest: optane
+ - name: Changing perm of "sh", adding "+x"
+ shell: "chmod +x setup-ca-kubernetes.sh"
+ args:
+ chdir: "optane"
+ warn: False
+ - copy:
+ src: "{{ playbook_dir }}/setup-ca.sh"
+ dest: optane
+ - name: Changing perm of "sh", adding "+x"
+ shell: "chmod +x setup-ca.sh"
+ args:
+ chdir: "optane"
+ warn: False
+ - copy:
+ src: "{{ playbook_dir }}/install_optane.sh"
+ dest: optane
+ - name: Changing perm of "sh", adding "+x"
+ shell: "chmod +x install_optane.sh"
+ args:
+ chdir: "optane"
+ warn: False
+ - copy:
+ src: "{{ playbook_dir }}/deploy_optane.sh"
+ dest: optane
+ - name: Changing perm of "sh", adding "+x"
+ shell: "chmod +x deploy_optane.sh"
+ args:
+ chdir: "optane"
+ warn: False
+ - copy:
+ src: "{{ playbook_dir }}/../images/pmem-csi-lvm.yaml"
+ dest: optane
+ - copy:
+ src: "{{ playbook_dir }}/../images/pmem-csi-direct.yaml"
+ dest: optane
+ - name: Run the install script and Re-evaluate the variable
+ command: optane/install_optane.sh
+ register: output
+ - name: restart kubelet services
+ become: yes
+ service:
+ name: kubelet
+ state: restarted
+
diff --git a/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh
new file mode 100755
index 00000000..c436f1ca
--- /dev/null
+++ b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh
@@ -0,0 +1,60 @@
+#!/bin/sh -e
+
+# This script generates certificates using setup-ca.sh and converts them into
+# the Kubernetes secrets that the PMEM-CSI deployments rely upon for
+# securing communication between PMEM-CSI components. Existing secrets
+# are updated with new certificates when running it again.
+
+# The script needs a functional kubectl that uses the target cluster.
+: ${KUBECTL:=kubectl}
+
+# The directory containing setup-ca*.sh.
+: ${TEST_DIRECTORY:=$(dirname $(readlink -f $0))}
+
+
+tmpdir=`mktemp -d`
+trap 'rm -r $tmpdir' EXIT
+
+# Generate certificates. They are not going to be needed again and will
+# be deleted together with the temp directory.
+WORKDIR="$tmpdir" "$TEST_DIRECTORY/setup-ca.sh"
+
+# This reads a file and encodes it for use in a secret.
+read_key () {
+ base64 -w 0 "$1"
+}
+
+# Read certificate files and turn them into Kubernetes secrets.
+#
+# -caFile (controller and all nodes)
+CA=$(read_key "$tmpdir/ca.pem")
+# -certFile (controller)
+REGISTRY_CERT=$(read_key "$tmpdir/pmem-registry.pem")
+# -keyFile (controller)
+REGISTRY_KEY=$(read_key "$tmpdir/pmem-registry-key.pem")
+# -certFile (same for all nodes)
+NODE_CERT=$(read_key "$tmpdir/pmem-node-controller.pem")
+# -keyFile (same for all nodes)
+NODE_KEY=$(read_key "$tmpdir/pmem-node-controller-key.pem")
+
+${KUBECTL} apply -f - <<EOF
+apiVersion: v1
+kind: Secret
+metadata:
+ name: pmem-csi-registry-secrets
+type: kubernetes.io/tls
+data:
+ ca.crt: ${CA}
+ tls.crt: ${REGISTRY_CERT}
+ tls.key: ${REGISTRY_KEY}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ name: pmem-csi-node-secrets
+type: Opaque
+data:
+ ca.crt: ${CA}
+ tls.crt: ${NODE_CERT}
+ tls.key: ${NODE_KEY}
+EOF
diff --git a/kud/deployment_infra/playbooks/setup-ca.sh b/kud/deployment_infra/playbooks/setup-ca.sh
new file mode 100755
index 00000000..77addc71
--- /dev/null
+++ b/kud/deployment_infra/playbooks/setup-ca.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# Directory to use for storing intermediate files.
+CA=${CA:="pmem-ca"}
+WORKDIR=${WORKDIR:-$(mktemp -d -u -t pmem-XXXX)}
+mkdir -p $WORKDIR
+cd $WORKDIR
+
+# Check for cfssl utilities.
+cfssl_found=1
+(command -v cfssl 2>&1 >/dev/null && command -v cfssljson 2>&1 >/dev/null) || cfssl_found=0
+if [ $cfssl_found -eq 0 ]; then
+ echo "cfssl tools not found, Please install cfssl and cfssljson."
+ exit 1
+fi
+
+# Generate CA certificates.
+<<EOF cfssl -loglevel=3 gencert -initca - | cfssljson -bare ca
+{
+ "CN": "$CA",
+ "key": {
+ "algo": "rsa",
+ "size": 2048
+ }
+}
+EOF
+
+# Generate server and client certificates.
+DEFAULT_CNS="pmem-registry pmem-node-controller"
+CNS="${DEFAULT_CNS} ${EXTRA_CNS:=""}"
+for name in ${CNS}; do
+ <<EOF cfssl -loglevel=3 gencert -ca=ca.pem -ca-key=ca-key.pem - | cfssljson -bare $name
+{
+ "CN": "$name",
+ "hosts": [
+ $(if [ "$name" = "pmem-registry" ]; then
+ # Some extra names needed for scheduler extender and webhook.
+ echo '"pmem-csi-scheduler", "pmem-csi-scheduler.default", "pmem-csi-scheduler.default.svc", "127.0.0.1",'
+ # And for metrics server.
+ echo '"pmem-csi-metrics", "pmem-csi-metrics.default", "pmem-csi-metrics.default.svc",'
+ fi
+ )
+ "$name"
+ ],
+ "key": {
+ "algo": "ecdsa",
+ "size": 256
+ }
+}
+EOF
+done