Fix QAT addon deploy and test

Note that as mentioned in install_qat.sh, the kernel command line must
include "intel_iommu=on iommu=pt" for the deploy and test to succeed.

The underlying issue is that the playbook was expecting to be run on
the same host it executed on and was looking for files in the wrong
places.

Issue-ID: MULTICLOUD-1261
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I5f59b9147f34f077fcdc63d7fc5f80b56977054c

5 files changed, 82 insertions, 103 deletions

diff --git a/kud/deployment_infra/playbooks/configure-qat.yml b/kud/deployment_infra/playbooks/configure-qat.yml
index 1225b3d4..39f52403 100644
--- a/kud/deployment_infra/playbooks/configure-qat.yml
+++ b/kud/deployment_infra/playbooks/configure-qat.yml
@@ -11,5 +11,5 @@
 - import_playbook: preconfigure-qat.yml
 - hosts: localhost
   tasks:
-    - name: Apply QAT plugin previleges Daemonset
+    - name: Apply QAT plugin privileges Daemonset
       command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/qat_plugin_privileges.yaml"
diff --git a/kud/deployment_infra/playbooks/install_qat.sh b/kud/deployment_infra/playbooks/install_qat.sh
index 57adb923..4a7fdef7 100644
--- a/kud/deployment_infra/playbooks/install_qat.sh
+++ b/kud/deployment_infra/playbooks/install_qat.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Precondition:
-# QAT device installed, such as lspci | grep 37c8
+# QAT device installed, such as lspci -n | grep 37c8
 # Enable grub with "intel_iommu=on iommu=pt"
 
 ROOT=
diff --git a/kud/deployment_infra/playbooks/preconfigure-qat.yml b/kud/deployment_infra/playbooks/preconfigure-qat.yml
index f5d797f1..ef8446f8 100644
--- a/kud/deployment_infra/playbooks/preconfigure-qat.yml
+++ b/kud/deployment_infra/playbooks/preconfigure-qat.yml
@@ -19,12 +19,10 @@
       file:
         state: directory
         path: "{{ qat_dest }}"
-    - name: Fetching QAT driver
-      block:
-      - name: Download QAT driver tarball
-        get_url:
-          url: "{{ qat_driver_url }}"
-          dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
+    - name: Download QAT driver tarball
+      get_url:
+        url: "{{ qat_driver_url }}"
+        dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
 
 - hosts: kube-node
   become: yes
@@ -33,21 +31,13 @@
       include_vars:
         file: kud-vars.yml
   tasks:
-    - name: Create a destination for driver folder in the target's /tmp
-      file:
-        state: directory
-        path: "{{ item }}"
-      with_items:
-        - "{{ base_dest }}/quick-assist/{{ qat_package }}"
-    - name: Create QAT dest folder
+    - name: Create destination folder for QAT check script
       file:
         state: directory
-        path: "qat"
-    - name: Register QAT env variable
-      shell: "echo {{ QAT_ENABLED | default(False) }}"
+        path: "{{ base_dest }}/qat"
     - name: Create QAT check script
       copy:
-        dest: "qat/qat.sh"
+        dest: "{{ base_dest }}/qat/qat.sh"
         content: |
             #!/bin/bash
             qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \
@@ -59,15 +49,11 @@
             else
                 echo "True"
             fi
-    - name: Changing perm of "sh", adding "+x"
-      shell: "chmod +x qat.sh"
-      args:
-        chdir: "qat"
-        warn: False
-    - name: Run the script and re-evaluate the variable.
-      command: "./qat.sh"
+        mode: 0755
+    - name: Run QAT check script and re-evaluate the variable
+      command: ./qat.sh
       args:
-        chdir: "qat"
+        chdir: "{{ base_dest }}/qat"
       register: output
     - debug:
         var: output.stdout_lines
@@ -75,70 +61,68 @@
         QAT_ENABLED: "{{ output.stdout }}"
     - debug:
         var: output
-    - name: Clean the script and folder.
+    - name: Clean QAT check script and folder
       file:
-        path: qat
+        path: "{{ base_dest }}/qat"
         state: absent
-    - name: bootstrap | install qat compilation packages
-      package:
-        name: "{{ item }}"
-        state: present
-      with_items:
-        - pciutils
-        - build-essential
-        - libudev-dev
-        - pkg-config
-      when: QAT_ENABLED
-    - copy:
-        src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
-        dest: "{{ base_dest }}/quick-assist"
-        remote_src: no
-      when: QAT_ENABLED
-    - name: Extract QAT source code
-      unarchive:
-        src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
-        dest: "{{ base_dest }}/quick-assist/{{ qat_package }}"
-      when: QAT_ENABLED
-    - name: Configure the target
-      command: ./configure --enable-icp-sriov=host
-      args:
-        chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}"
-      when: QAT_ENABLED
-    - name: build qat driver
-      make:
-        chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}"
-        target: "{{ item }}"
-      loop:
-        - clean
-        - uninstall
-        - install
-      when: QAT_ENABLED
-    - name: Create QAT driver folder in the target destination
-      file:
-        state: directory
-        path: "{{ item }}"
-      with_items:
-        - qat_driver_dest
-      when: QAT_ENABLED
-    - name: Copy QAT build directory qat target destination
-      command: "cp -r {{ base_dest }}/quick-assist/{{ qat_package }}/build/ /root/qat_driver_dest/"
-      when: QAT_ENABLED
-    - name: Copy QAT driver install script to target folder
-      command: "cp {{ playbook_dir }}/install_qat.sh /root/qat_driver_dest/build/install.sh"
-      when: QAT_ENABLED
-    - name: Copy QAT to target folder
-      command: "cp /etc/default/qat /root/qat_driver_dest/build"
-      when: QAT_ENABLED
-    - name: Changing perm of "install.sh", adding "+x"
-      file: dest=~/qat_driver_dest/build/install.sh mode=a+x
-      when: QAT_ENABLED
-    - name: Run a script with arguments
-      command: ./install.sh chdir=/root/qat_driver_dest/build
-      when: QAT_ENABLED
-    - name: get qat devices
-      shell: /usr/local/bin/adf_ctl status | grep up | awk '{print $4 substr($1, 4)}' | tr -d ','
-      register: qat_devices
-      when: QAT_ENABLED
-    - name: Updating the qat device SSL values to avoid duplication
-      command: "./substitute.sh chdir={{ playbook_dir }}"
+    - name: Install QAT driver
+      block:
+      - name: Install QAT compilation packages
+        package:
+          name: "{{ item }}"
+          state: present
+        with_items:
+          - pciutils
+          - build-essential
+          - libudev-dev
+          - pkg-config
+      - name: Create destination folder for QAT source code
+        file:
+          state: directory
+          path: "{{ qat_dest }}/{{ qat_package }}"
+      - name: Extract QAT source code
+        unarchive:
+          src: "{{ qat_dest }}/{{ qat_package }}.tar.gz"
+          dest: "{{ qat_dest }}/{{ qat_package }}"
+      - name: Configure the target
+        command: ./configure --enable-icp-sriov=host
+        args:
+          chdir: "{{ qat_dest }}/{{ qat_package }}"
+      - name: Build QAT driver
+        make:
+          chdir: "{{ qat_dest }}/{{ qat_package }}"
+          target: "{{ item }}"
+        loop:
+          - clean
+          - uninstall
+          - install
+      - name: Copy QAT driver install script to target folder
+        copy:
+          src: "install_qat.sh"
+          dest: "{{ qat_dest }}/{{ qat_package }}/build"
+          mode: 0755
+      - name: Copy /etc/default/qat to target folder
+        copy:
+          src: "/etc/default/qat"
+          dest: "{{ qat_dest }}/{{ qat_package }}/build"
+          remote_src: yes
+      - name: Run a script with arguments
+        command: ./install_qat.sh
+        args:
+          chdir: "{{ qat_dest }}/{{ qat_package }}/build"
+      - name: Copy QAT substitue script to target folder
+        copy:
+          src: "substitute.sh"
+          dest: "{{ qat_dest }}/{{ qat_package }}/build"
+          mode: 0755
+      - name: Update the QAT device SSL values to avoid duplication
+        command: ./substitute.sh
+        args:
+          chdir: "{{ qat_dest }}/{{ qat_package }}/build"
+      - name: Restart acceleration driver framework
+        command: adf_ctl restart
+      - name: Restart QAT service
+        service:
+          name: qat_service
+          state: restarted
       when: QAT_ENABLED
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index 226f4568..db6b224c 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -122,7 +122,7 @@ function install_addons {
         tee $cluster_log/setup-kud.log
     # The order of KUD_ADDONS is important: some plugins (sriov, qat)
     # require nfd to be enabled.
-    for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
+    for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
         echo "Deploying $addon using configure-$addon.yml playbook.."
         ansible-playbook $verbose -i \
             $kud_inventory $kud_playbooks/configure-${addon}.yml | \
@@ -131,7 +131,7 @@ function install_addons {
 
     echo "Run the test cases if testing_enabled is set to true."
     if [[ "${testing_enabled}" == "true" ]]; then
-        for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
+        for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
             pushd $kud_tests
             bash ${addon}.sh
             case $addon in
diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh
index 2f8d212a..8365f700 100755
--- a/kud/tests/qat.sh
+++ b/kud/tests/qat.sh
@@ -10,16 +10,13 @@
 
 set -o pipefail
 
-qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \
-                do lspci -d 8086:$i -m; done |\
-                grep -i "Quick*" | head -n 1 | cut -d " " -f 5 )
-#Checking if the QAT device is on the node
-if [ -z "$qat_device" ]; then
-    echo "False. This test case cannot run. Qat device unavailable."
+qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity."qat.intel.com/cy2_dc2">="1") | .metadata.name')
+if [ -z "$qat_capable_nodes" ]; then
+    echo "This test case cannot run. QAT device unavailable."
     QAT_ENABLED=False
     exit 0
 else
-    echo "True. Can run QAT on this device."
+    echo "Can run QAT on this cluster."
     QAT_ENABLED=True
 fi
 
@@ -78,9 +75,7 @@ kubectl create -f $HOME/$pod_name.yaml --validate=false
 
 allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}')
 echo "The allocated resource of the node is: " $allocated_node_resource
-adf_ctl restart
-systemctl restart qat_service
-kubectl exec -it pod-case-01 -- openssl engine -c -t qat
+kubectl exec pod-case-01 -- openssl engine -c -t qat
 
 kubectl delete pod $pod_name --now
 echo "Test complete."