From d3b0887248f10248d4a87a91f918984761f95599 Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Wed, 2 Dec 2020 16:09:36 -0800 Subject: Fix QAT addon deploy and test Note that as mentioned in install_qat.sh, the kernel command line must include "intel_iommu=on iommu=pt" for the deploy and test to succeed. The underlying issue is that the playbook was expecting to be run on the same host it executed on and was looking for files in the wrong places. Issue-ID: MULTICLOUD-1261 Signed-off-by: Todd Malsbary Change-Id: I5f59b9147f34f077fcdc63d7fc5f80b56977054c --- kud/deployment_infra/playbooks/configure-qat.yml | 2 +- kud/deployment_infra/playbooks/install_qat.sh | 2 +- .../playbooks/preconfigure-qat.yml | 162 ++++++++++----------- kud/hosting_providers/containerized/installer.sh | 4 +- kud/tests/qat.sh | 15 +- 5 files changed, 82 insertions(+), 103 deletions(-) (limited to 'kud') diff --git a/kud/deployment_infra/playbooks/configure-qat.yml b/kud/deployment_infra/playbooks/configure-qat.yml index 1225b3d4..39f52403 100644 --- a/kud/deployment_infra/playbooks/configure-qat.yml +++ b/kud/deployment_infra/playbooks/configure-qat.yml @@ -11,5 +11,5 @@ - import_playbook: preconfigure-qat.yml - hosts: localhost tasks: - - name: Apply QAT plugin previleges Daemonset + - name: Apply QAT plugin privileges Daemonset command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/qat_plugin_privileges.yaml" diff --git a/kud/deployment_infra/playbooks/install_qat.sh b/kud/deployment_infra/playbooks/install_qat.sh index 57adb923..4a7fdef7 100644 --- a/kud/deployment_infra/playbooks/install_qat.sh +++ b/kud/deployment_infra/playbooks/install_qat.sh @@ -1,7 +1,7 @@ #!/bin/bash # Precondition: -# QAT device installed, such as lspci | grep 37c8 +# QAT device installed, such as lspci -n | grep 37c8 # Enable grub with "intel_iommu=on iommu=pt" ROOT= diff --git a/kud/deployment_infra/playbooks/preconfigure-qat.yml b/kud/deployment_infra/playbooks/preconfigure-qat.yml index f5d797f1..ef8446f8 100644 --- a/kud/deployment_infra/playbooks/preconfigure-qat.yml +++ b/kud/deployment_infra/playbooks/preconfigure-qat.yml @@ -19,12 +19,10 @@ file: state: directory path: "{{ qat_dest }}" - - name: Fetching QAT driver - block: - - name: Download QAT driver tarball - get_url: - url: "{{ qat_driver_url }}" - dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + - name: Download QAT driver tarball + get_url: + url: "{{ qat_driver_url }}" + dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - hosts: kube-node become: yes @@ -33,21 +31,13 @@ include_vars: file: kud-vars.yml tasks: - - name: Create a destination for driver folder in the target's /tmp - file: - state: directory - path: "{{ item }}" - with_items: - - "{{ base_dest }}/quick-assist/{{ qat_package }}" - - name: Create QAT dest folder + - name: Create destination folder for QAT check script file: state: directory - path: "qat" - - name: Register QAT env variable - shell: "echo {{ QAT_ENABLED | default(False) }}" + path: "{{ base_dest }}/qat" - name: Create QAT check script copy: - dest: "qat/qat.sh" + dest: "{{ base_dest }}/qat/qat.sh" content: | #!/bin/bash qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \ @@ -59,15 +49,11 @@ else echo "True" fi - - name: Changing perm of "sh", adding "+x" - shell: "chmod +x qat.sh" - args: - chdir: "qat" - warn: False - - name: Run the script and re-evaluate the variable. - command: "./qat.sh" + mode: 0755 + - name: Run QAT check script and re-evaluate the variable + command: ./qat.sh args: - chdir: "qat" + chdir: "{{ base_dest }}/qat" register: output - debug: var: output.stdout_lines @@ -75,70 +61,68 @@ QAT_ENABLED: "{{ output.stdout }}" - debug: var: output - - name: Clean the script and folder. + - name: Clean QAT check script and folder file: - path: qat + path: "{{ base_dest }}/qat" state: absent - - name: bootstrap | install qat compilation packages - package: - name: "{{ item }}" - state: present - with_items: - - pciutils - - build-essential - - libudev-dev - - pkg-config - when: QAT_ENABLED - - copy: - src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - dest: "{{ base_dest }}/quick-assist" - remote_src: no - when: QAT_ENABLED - - name: Extract QAT source code - unarchive: - src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - dest: "{{ base_dest }}/quick-assist/{{ qat_package }}" - when: QAT_ENABLED - - name: Configure the target - command: ./configure --enable-icp-sriov=host - args: - chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" - when: QAT_ENABLED - - name: build qat driver - make: - chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" - target: "{{ item }}" - loop: - - clean - - uninstall - - install - when: QAT_ENABLED - - name: Create QAT driver folder in the target destination - file: - state: directory - path: "{{ item }}" - with_items: - - qat_driver_dest - when: QAT_ENABLED - - name: Copy QAT build directory qat target destination - command: "cp -r {{ base_dest }}/quick-assist/{{ qat_package }}/build/ /root/qat_driver_dest/" - when: QAT_ENABLED - - name: Copy QAT driver install script to target folder - command: "cp {{ playbook_dir }}/install_qat.sh /root/qat_driver_dest/build/install.sh" - when: QAT_ENABLED - - name: Copy QAT to target folder - command: "cp /etc/default/qat /root/qat_driver_dest/build" - when: QAT_ENABLED - - name: Changing perm of "install.sh", adding "+x" - file: dest=~/qat_driver_dest/build/install.sh mode=a+x - when: QAT_ENABLED - - name: Run a script with arguments - command: ./install.sh chdir=/root/qat_driver_dest/build - when: QAT_ENABLED - - name: get qat devices - shell: /usr/local/bin/adf_ctl status | grep up | awk '{print $4 substr($1, 4)}' | tr -d ',' - register: qat_devices - when: QAT_ENABLED - - name: Updating the qat device SSL values to avoid duplication - command: "./substitute.sh chdir={{ playbook_dir }}" + - name: Install QAT driver + block: + - name: Install QAT compilation packages + package: + name: "{{ item }}" + state: present + with_items: + - pciutils + - build-essential + - libudev-dev + - pkg-config + - name: Create destination folder for QAT source code + file: + state: directory + path: "{{ qat_dest }}/{{ qat_package }}" + - name: Extract QAT source code + unarchive: + src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + dest: "{{ qat_dest }}/{{ qat_package }}" + - name: Configure the target + command: ./configure --enable-icp-sriov=host + args: + chdir: "{{ qat_dest }}/{{ qat_package }}" + - name: Build QAT driver + make: + chdir: "{{ qat_dest }}/{{ qat_package }}" + target: "{{ item }}" + loop: + - clean + - uninstall + - install + - name: Copy QAT driver install script to target folder + copy: + src: "install_qat.sh" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + mode: 0755 + - name: Copy /etc/default/qat to target folder + copy: + src: "/etc/default/qat" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + remote_src: yes + - name: Run a script with arguments + command: ./install_qat.sh + args: + chdir: "{{ qat_dest }}/{{ qat_package }}/build" + - name: Copy QAT substitue script to target folder + copy: + src: "substitute.sh" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + mode: 0755 + - name: Update the QAT device SSL values to avoid duplication + command: ./substitute.sh + args: + chdir: "{{ qat_dest }}/{{ qat_package }}/build" + - name: Restart acceleration driver framework + command: adf_ctl restart + - name: Restart QAT service + service: + name: qat_service + state: restarted when: QAT_ENABLED diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 226f4568..db6b224c 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -122,7 +122,7 @@ function install_addons { tee $cluster_log/setup-kud.log # The order of KUD_ADDONS is important: some plugins (sriov, qat) # require nfd to be enabled. - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-${addon}.yml | \ @@ -131,7 +131,7 @@ function install_addons { echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do pushd $kud_tests bash ${addon}.sh case $addon in diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh index 2f8d212a..8365f700 100755 --- a/kud/tests/qat.sh +++ b/kud/tests/qat.sh @@ -10,16 +10,13 @@ set -o pipefail -qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \ - do lspci -d 8086:$i -m; done |\ - grep -i "Quick*" | head -n 1 | cut -d " " -f 5 ) -#Checking if the QAT device is on the node -if [ -z "$qat_device" ]; then - echo "False. This test case cannot run. Qat device unavailable." +qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity."qat.intel.com/cy2_dc2">="1") | .metadata.name') +if [ -z "$qat_capable_nodes" ]; then + echo "This test case cannot run. QAT device unavailable." QAT_ENABLED=False exit 0 else - echo "True. Can run QAT on this device." + echo "Can run QAT on this cluster." QAT_ENABLED=True fi @@ -78,9 +75,7 @@ kubectl create -f $HOME/$pod_name.yaml --validate=false allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}') echo "The allocated resource of the node is: " $allocated_node_resource -adf_ctl restart -systemctl restart qat_service -kubectl exec -it pod-case-01 -- openssl engine -c -t qat +kubectl exec pod-case-01 -- openssl engine -c -t qat kubectl delete pod $pod_name --now echo "Test complete." -- cgit 1.2.3-korg