summaryrefslogtreecommitdiffstats
path: root/kud
diff options
context:
space:
mode:
authorTodd Malsbary <todd.malsbary@intel.com>2020-12-02 16:09:36 -0800
committerTodd Malsbary <todd.malsbary@intel.com>2020-12-07 16:41:54 -0800
commitd3b0887248f10248d4a87a91f918984761f95599 (patch)
tree7fd1e5aace04f3e940c56d66ff2d47be10dce5ef /kud
parent7e06fbaa3d1293ca9b25aeb7ea7cb7be2179e30a (diff)
Fix QAT addon deploy and test
Note that as mentioned in install_qat.sh, the kernel command line must include "intel_iommu=on iommu=pt" for the deploy and test to succeed. The underlying issue is that the playbook was expecting to be run on the same host it executed on and was looking for files in the wrong places. Issue-ID: MULTICLOUD-1261 Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> Change-Id: I5f59b9147f34f077fcdc63d7fc5f80b56977054c
Diffstat (limited to 'kud')
-rw-r--r--kud/deployment_infra/playbooks/configure-qat.yml2
-rw-r--r--kud/deployment_infra/playbooks/install_qat.sh2
-rw-r--r--kud/deployment_infra/playbooks/preconfigure-qat.yml162
-rwxr-xr-xkud/hosting_providers/containerized/installer.sh4
-rwxr-xr-xkud/tests/qat.sh15
5 files changed, 82 insertions, 103 deletions
diff --git a/kud/deployment_infra/playbooks/configure-qat.yml b/kud/deployment_infra/playbooks/configure-qat.yml
index 1225b3d4..39f52403 100644
--- a/kud/deployment_infra/playbooks/configure-qat.yml
+++ b/kud/deployment_infra/playbooks/configure-qat.yml
@@ -11,5 +11,5 @@
- import_playbook: preconfigure-qat.yml
- hosts: localhost
tasks:
- - name: Apply QAT plugin previleges Daemonset
+ - name: Apply QAT plugin privileges Daemonset
command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/qat_plugin_privileges.yaml"
diff --git a/kud/deployment_infra/playbooks/install_qat.sh b/kud/deployment_infra/playbooks/install_qat.sh
index 57adb923..4a7fdef7 100644
--- a/kud/deployment_infra/playbooks/install_qat.sh
+++ b/kud/deployment_infra/playbooks/install_qat.sh
@@ -1,7 +1,7 @@
#!/bin/bash
# Precondition:
-# QAT device installed, such as lspci | grep 37c8
+# QAT device installed, such as lspci -n | grep 37c8
# Enable grub with "intel_iommu=on iommu=pt"
ROOT=
diff --git a/kud/deployment_infra/playbooks/preconfigure-qat.yml b/kud/deployment_infra/playbooks/preconfigure-qat.yml
index f5d797f1..ef8446f8 100644
--- a/kud/deployment_infra/playbooks/preconfigure-qat.yml
+++ b/kud/deployment_infra/playbooks/preconfigure-qat.yml
@@ -19,12 +19,10 @@
file:
state: directory
path: "{{ qat_dest }}"
- - name: Fetching QAT driver
- block:
- - name: Download QAT driver tarball
- get_url:
- url: "{{ qat_driver_url }}"
- dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
+ - name: Download QAT driver tarball
+ get_url:
+ url: "{{ qat_driver_url }}"
+ dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
- hosts: kube-node
become: yes
@@ -33,21 +31,13 @@
include_vars:
file: kud-vars.yml
tasks:
- - name: Create a destination for driver folder in the target's /tmp
- file:
- state: directory
- path: "{{ item }}"
- with_items:
- - "{{ base_dest }}/quick-assist/{{ qat_package }}"
- - name: Create QAT dest folder
+ - name: Create destination folder for QAT check script
file:
state: directory
- path: "qat"
- - name: Register QAT env variable
- shell: "echo {{ QAT_ENABLED | default(False) }}"
+ path: "{{ base_dest }}/qat"
- name: Create QAT check script
copy:
- dest: "qat/qat.sh"
+ dest: "{{ base_dest }}/qat/qat.sh"
content: |
#!/bin/bash
qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \
@@ -59,15 +49,11 @@
else
echo "True"
fi
- - name: Changing perm of "sh", adding "+x"
- shell: "chmod +x qat.sh"
- args:
- chdir: "qat"
- warn: False
- - name: Run the script and re-evaluate the variable.
- command: "./qat.sh"
+ mode: 0755
+ - name: Run QAT check script and re-evaluate the variable
+ command: ./qat.sh
args:
- chdir: "qat"
+ chdir: "{{ base_dest }}/qat"
register: output
- debug:
var: output.stdout_lines
@@ -75,70 +61,68 @@
QAT_ENABLED: "{{ output.stdout }}"
- debug:
var: output
- - name: Clean the script and folder.
+ - name: Clean QAT check script and folder
file:
- path: qat
+ path: "{{ base_dest }}/qat"
state: absent
- - name: bootstrap | install qat compilation packages
- package:
- name: "{{ item }}"
- state: present
- with_items:
- - pciutils
- - build-essential
- - libudev-dev
- - pkg-config
- when: QAT_ENABLED
- - copy:
- src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
- dest: "{{ base_dest }}/quick-assist"
- remote_src: no
- when: QAT_ENABLED
- - name: Extract QAT source code
- unarchive:
- src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
- dest: "{{ base_dest }}/quick-assist/{{ qat_package }}"
- when: QAT_ENABLED
- - name: Configure the target
- command: ./configure --enable-icp-sriov=host
- args:
- chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}"
- when: QAT_ENABLED
- - name: build qat driver
- make:
- chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}"
- target: "{{ item }}"
- loop:
- - clean
- - uninstall
- - install
- when: QAT_ENABLED
- - name: Create QAT driver folder in the target destination
- file:
- state: directory
- path: "{{ item }}"
- with_items:
- - qat_driver_dest
- when: QAT_ENABLED
- - name: Copy QAT build directory qat target destination
- command: "cp -r {{ base_dest }}/quick-assist/{{ qat_package }}/build/ /root/qat_driver_dest/"
- when: QAT_ENABLED
- - name: Copy QAT driver install script to target folder
- command: "cp {{ playbook_dir }}/install_qat.sh /root/qat_driver_dest/build/install.sh"
- when: QAT_ENABLED
- - name: Copy QAT to target folder
- command: "cp /etc/default/qat /root/qat_driver_dest/build"
- when: QAT_ENABLED
- - name: Changing perm of "install.sh", adding "+x"
- file: dest=~/qat_driver_dest/build/install.sh mode=a+x
- when: QAT_ENABLED
- - name: Run a script with arguments
- command: ./install.sh chdir=/root/qat_driver_dest/build
- when: QAT_ENABLED
- - name: get qat devices
- shell: /usr/local/bin/adf_ctl status | grep up | awk '{print $4 substr($1, 4)}' | tr -d ','
- register: qat_devices
- when: QAT_ENABLED
- - name: Updating the qat device SSL values to avoid duplication
- command: "./substitute.sh chdir={{ playbook_dir }}"
+ - name: Install QAT driver
+ block:
+ - name: Install QAT compilation packages
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items:
+ - pciutils
+ - build-essential
+ - libudev-dev
+ - pkg-config
+ - name: Create destination folder for QAT source code
+ file:
+ state: directory
+ path: "{{ qat_dest }}/{{ qat_package }}"
+ - name: Extract QAT source code
+ unarchive:
+ src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
+ dest: "{{ qat_dest }}/{{ qat_package }}"
+ - name: Configure the target
+ command: ./configure --enable-icp-sriov=host
+ args:
+ chdir: "{{ qat_dest }}/{{ qat_package }}"
+ - name: Build QAT driver
+ make:
+ chdir: "{{ qat_dest }}/{{ qat_package }}"
+ target: "{{ item }}"
+ loop:
+ - clean
+ - uninstall
+ - install
+ - name: Copy QAT driver install script to target folder
+ copy:
+ src: "install_qat.sh"
+ dest: "{{ qat_dest }}/{{ qat_package }}/build"
+ mode: 0755
+ - name: Copy /etc/default/qat to target folder
+ copy:
+ src: "/etc/default/qat"
+ dest: "{{ qat_dest }}/{{ qat_package }}/build"
+ remote_src: yes
+ - name: Run a script with arguments
+ command: ./install_qat.sh
+ args:
+ chdir: "{{ qat_dest }}/{{ qat_package }}/build"
+ - name: Copy QAT substitue script to target folder
+ copy:
+ src: "substitute.sh"
+ dest: "{{ qat_dest }}/{{ qat_package }}/build"
+ mode: 0755
+ - name: Update the QAT device SSL values to avoid duplication
+ command: ./substitute.sh
+ args:
+ chdir: "{{ qat_dest }}/{{ qat_package }}/build"
+ - name: Restart acceleration driver framework
+ command: adf_ctl restart
+ - name: Restart QAT service
+ service:
+ name: qat_service
+ state: restarted
when: QAT_ENABLED
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index 226f4568..db6b224c 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -122,7 +122,7 @@ function install_addons {
tee $cluster_log/setup-kud.log
# The order of KUD_ADDONS is important: some plugins (sriov, qat)
# require nfd to be enabled.
- for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
+ for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
echo "Deploying $addon using configure-$addon.yml playbook.."
ansible-playbook $verbose -i \
$kud_inventory $kud_playbooks/configure-${addon}.yml | \
@@ -131,7 +131,7 @@ function install_addons {
echo "Run the test cases if testing_enabled is set to true."
if [[ "${testing_enabled}" == "true" ]]; then
- for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
+ for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
pushd $kud_tests
bash ${addon}.sh
case $addon in
diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh
index 2f8d212a..8365f700 100755
--- a/kud/tests/qat.sh
+++ b/kud/tests/qat.sh
@@ -10,16 +10,13 @@
set -o pipefail
-qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \
- do lspci -d 8086:$i -m; done |\
- grep -i "Quick*" | head -n 1 | cut -d " " -f 5 )
-#Checking if the QAT device is on the node
-if [ -z "$qat_device" ]; then
- echo "False. This test case cannot run. Qat device unavailable."
+qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity."qat.intel.com/cy2_dc2">="1") | .metadata.name')
+if [ -z "$qat_capable_nodes" ]; then
+ echo "This test case cannot run. QAT device unavailable."
QAT_ENABLED=False
exit 0
else
- echo "True. Can run QAT on this device."
+ echo "Can run QAT on this cluster."
QAT_ENABLED=True
fi
@@ -78,9 +75,7 @@ kubectl create -f $HOME/$pod_name.yaml --validate=false
allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}')
echo "The allocated resource of the node is: " $allocated_node_resource
-adf_ctl restart
-systemctl restart qat_service
-kubectl exec -it pod-case-01 -- openssl engine -c -t qat
+kubectl exec pod-case-01 -- openssl engine -c -t qat
kubectl delete pod $pod_name --now
echo "Test complete."