From 22f755508a107c689f325042c4abaa98c5bd450e Mon Sep 17 00:00:00 2001 From: "Chen, Tingjie" Date: Mon, 25 May 2020 14:26:26 +0800 Subject: Add support for pmem-csi plugin and e2e test Issue-ID: MULTICLOUD-1046 Change-Id: I1853e071a99702c5e6f7ba9ca819746576fd0aca Signed-off-by: Chen, Tingjie --- .../playbooks/configure-optane.yml | 15 ++++ kud/deployment_infra/playbooks/deploy_optane.sh | 10 +++ kud/deployment_infra/playbooks/install_optane.sh | 71 ++++++++++++++++++ kud/deployment_infra/playbooks/kud-vars.yml | 6 ++ .../playbooks/preconfigure-optane.yml | 85 ++++++++++++++++++++++ .../playbooks/setup-ca-kubernetes.sh | 60 +++++++++++++++ kud/deployment_infra/playbooks/setup-ca.sh | 51 +++++++++++++ 7 files changed, 298 insertions(+) create mode 100644 kud/deployment_infra/playbooks/configure-optane.yml create mode 100755 kud/deployment_infra/playbooks/deploy_optane.sh create mode 100755 kud/deployment_infra/playbooks/install_optane.sh create mode 100644 kud/deployment_infra/playbooks/preconfigure-optane.yml create mode 100755 kud/deployment_infra/playbooks/setup-ca-kubernetes.sh create mode 100755 kud/deployment_infra/playbooks/setup-ca.sh (limited to 'kud/deployment_infra/playbooks') diff --git a/kud/deployment_infra/playbooks/configure-optane.yml b/kud/deployment_infra/playbooks/configure-optane.yml new file mode 100644 index 00000000..8e000aa4 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-optane.yml @@ -0,0 +1,15 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- import_playbook: preconfigure-optane.yml +- hosts: localhost + tasks: + - name: Apply Optane PMEM CSI Daemonset + command: "{{ base_dest }}/optane/deploy_optane.sh" diff --git a/kud/deployment_infra/playbooks/deploy_optane.sh b/kud/deployment_infra/playbooks/deploy_optane.sh new file mode 100755 index 00000000..cb502373 --- /dev/null +++ b/kud/deployment_infra/playbooks/deploy_optane.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +work_path="$(dirname -- "$(readlink -f -- "$0")")" +ndctl_region=`ndctl list -R` +if [[ $ndctl_region == "" ]] ; then + echo "No Optane Hardware!" +else + echo "Optane Plugin start .." + /usr/local/bin/kubectl apply -f $work_path/pmem-csi-lvm.yaml +fi diff --git a/kud/deployment_infra/playbooks/install_optane.sh b/kud/deployment_infra/playbooks/install_optane.sh new file mode 100755 index 00000000..3f345362 --- /dev/null +++ b/kud/deployment_infra/playbooks/install_optane.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# Precondition: +# Optane PM related utilities download and configure. + +# collet and install ndctl and check hardware +echo "[OPTANE] Install ndctl ..." +apt install -y ndctl + +echo "[OPTANE] Check the NVDIMM hardware ..." +ndctl_region=`ndctl list -R` +if [[ $ndctl_region == "" ]] ; then + echo "No NVDIMM hardware, exit ..." + exit 0 +fi + +# get current folder path +work_path="$(dirname -- "$(readlink -f -- "$0")")" +node_name="$(kubectl get node -o jsonpath='{.items[0].metadata.name}')" + +# collet and install ipmctl +echo "[OPTANE] Install ipmctl ..." +cd $work_path +wget https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/02.00.00.3474+really01.00.00.3469-1/ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz +tar xvf ipmctl_02.00.00.3474+really01.00.00.3469.orig.tar.xz +cd ipmctl-01.00.00.3469/ + +echo "[OPTANE] Install ipmctl utilities" +mkdir output && cd output +apt install -y cmake build-essential pkg-config asciidoctor asciidoc libndctl-dev git +gem install asciidoctor-pdf --pre + +add-apt-repository --yes ppa:jhli/libsafec +apt update +apt-get install -y libsafec-dev + +echo "[OPTANE] Build ipmctl ..." +cmake -DRELEASE=ON -DCMAKE_INSTALL_PREFIX=/ .. +make -j all +make install + +cd $work_path + +echo "[OPTANE] Install cfssl tools ..." +# collect cfssl tools +curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o cfssl +curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o cfssljson +chmod a+x cfssl cfssljson +cp -rf cfssl cfssljson /usr/bin/ + +echo "[OPTANE] Create AppDirect Goal ..." +# ipmctl setting +#ipmctl delete -goal +#ipmctl create -f -goal PersistentMemoryType=AppDirectNotInterleaved + +# Run certificates set-up script +echo "[OPTANE] Run ca for kubernetes ..." +./setup-ca-kubernetes.sh + +# deploy docker hub +echo "[OPTANE] Set label node for storage pmem ..." +kubectl label node $node_name storage=pmem + +echo "[OPTANE] kubelet CSIMigration set false ..." +echo -e "featureGates:\n CSIMigration: false" >> /var/lib/kubelet/config.yaml +# deploy pmem-csi and applications +# select two mode: lvm and direct +#echo "[OPTANE] Create PMEM-CSI plugin service ..." +#kubectl create -f ../images/pmem-csi-lvm.yaml +# kubectl create -f pmem-csi-direct.yaml + diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 7a25670d..e5438ae4 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -73,3 +73,9 @@ qat_driver_source_type: "tarball" qat_driver_version: 1.7.l.4.6.0-00025 qat_driver_url: "https://01.org/sites/default/files/downloads/{{ qat_package }}.tar.gz" qat_package: qat1.7.l.4.6.0-00025 + +optane_dest: "{{ base_dest }}/optane" +optane_ipmctl_source_type: "tarball" +optane_ipmctl_version: 02.00.00.3474 +optane_ipmctl_url: "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/{{ optane_package }}.tar.xz" +optane_ipmctl_package: ipmctl_02.00.00.3474+really01.00.00.3469.orig diff --git a/kud/deployment_infra/playbooks/preconfigure-optane.yml b/kud/deployment_infra/playbooks/preconfigure-optane.yml new file mode 100644 index 00000000..64622895 --- /dev/null +++ b/kud/deployment_infra/playbooks/preconfigure-optane.yml @@ -0,0 +1,85 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Run the following task only if the SRIOV is set to True +# i.e when SRIOV hardware is available +- hosts: localhost + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: Create optane folder + file: + state: directory + path: "{{ optane_dest }}" + ignore_errors: yes + +- hosts: kube-node + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: Create OPTANE folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - optane + - copy: + src: "{{ playbook_dir }}/setup-ca-kubernetes.sh" + dest: optane + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x setup-ca-kubernetes.sh" + args: + chdir: "optane" + warn: False + - copy: + src: "{{ playbook_dir }}/setup-ca.sh" + dest: optane + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x setup-ca.sh" + args: + chdir: "optane" + warn: False + - copy: + src: "{{ playbook_dir }}/install_optane.sh" + dest: optane + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x install_optane.sh" + args: + chdir: "optane" + warn: False + - copy: + src: "{{ playbook_dir }}/deploy_optane.sh" + dest: optane + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x deploy_optane.sh" + args: + chdir: "optane" + warn: False + - copy: + src: "{{ playbook_dir }}/../images/pmem-csi-lvm.yaml" + dest: optane + - copy: + src: "{{ playbook_dir }}/../images/pmem-csi-direct.yaml" + dest: optane + - name: Run the install script and Re-evaluate the variable + command: optane/install_optane.sh + register: output + - name: restart kubelet services + become: yes + service: + name: kubelet + state: restarted + diff --git a/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh new file mode 100755 index 00000000..c436f1ca --- /dev/null +++ b/kud/deployment_infra/playbooks/setup-ca-kubernetes.sh @@ -0,0 +1,60 @@ +#!/bin/sh -e + +# This script generates certificates using setup-ca.sh and converts them into +# the Kubernetes secrets that the PMEM-CSI deployments rely upon for +# securing communication between PMEM-CSI components. Existing secrets +# are updated with new certificates when running it again. + +# The script needs a functional kubectl that uses the target cluster. +: ${KUBECTL:=kubectl} + +# The directory containing setup-ca*.sh. +: ${TEST_DIRECTORY:=$(dirname $(readlink -f $0))} + + +tmpdir=`mktemp -d` +trap 'rm -r $tmpdir' EXIT + +# Generate certificates. They are not going to be needed again and will +# be deleted together with the temp directory. +WORKDIR="$tmpdir" "$TEST_DIRECTORY/setup-ca.sh" + +# This reads a file and encodes it for use in a secret. +read_key () { + base64 -w 0 "$1" +} + +# Read certificate files and turn them into Kubernetes secrets. +# +# -caFile (controller and all nodes) +CA=$(read_key "$tmpdir/ca.pem") +# -certFile (controller) +REGISTRY_CERT=$(read_key "$tmpdir/pmem-registry.pem") +# -keyFile (controller) +REGISTRY_KEY=$(read_key "$tmpdir/pmem-registry-key.pem") +# -certFile (same for all nodes) +NODE_CERT=$(read_key "$tmpdir/pmem-node-controller.pem") +# -keyFile (same for all nodes) +NODE_KEY=$(read_key "$tmpdir/pmem-node-controller-key.pem") + +${KUBECTL} apply -f - <&1 >/dev/null && command -v cfssljson 2>&1 >/dev/null) || cfssl_found=0 +if [ $cfssl_found -eq 0 ]; then + echo "cfssl tools not found, Please install cfssl and cfssljson." + exit 1 +fi + +# Generate CA certificates. +<