diff options
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-qat.yml | 2 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/install_qat.sh | 2 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/preconfigure-qat.yml | 162 | ||||
| -rwxr-xr-x | kud/hosting_providers/containerized/installer.sh | 4 | ||||
| -rwxr-xr-x | kud/tests/qat.sh | 15 |
5 files changed, 82 insertions, 103 deletions
diff --git a/kud/deployment_infra/playbooks/configure-qat.yml b/kud/deployment_infra/playbooks/configure-qat.yml index 1225b3d4..39f52403 100644 --- a/kud/deployment_infra/playbooks/configure-qat.yml +++ b/kud/deployment_infra/playbooks/configure-qat.yml @@ -11,5 +11,5 @@ - import_playbook: preconfigure-qat.yml - hosts: localhost tasks: - - name: Apply QAT plugin previleges Daemonset + - name: Apply QAT plugin privileges Daemonset command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/qat_plugin_privileges.yaml" diff --git a/kud/deployment_infra/playbooks/install_qat.sh b/kud/deployment_infra/playbooks/install_qat.sh index 57adb923..4a7fdef7 100644 --- a/kud/deployment_infra/playbooks/install_qat.sh +++ b/kud/deployment_infra/playbooks/install_qat.sh @@ -1,7 +1,7 @@ #!/bin/bash # Precondition: -# QAT device installed, such as lspci | grep 37c8 +# QAT device installed, such as lspci -n | grep 37c8 # Enable grub with "intel_iommu=on iommu=pt" ROOT= diff --git a/kud/deployment_infra/playbooks/preconfigure-qat.yml b/kud/deployment_infra/playbooks/preconfigure-qat.yml index f5d797f1..ef8446f8 100644 --- a/kud/deployment_infra/playbooks/preconfigure-qat.yml +++ b/kud/deployment_infra/playbooks/preconfigure-qat.yml @@ -19,12 +19,10 @@ file: state: directory path: "{{ qat_dest }}" - - name: Fetching QAT driver - block: - - name: Download QAT driver tarball - get_url: - url: "{{ qat_driver_url }}" - dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + - name: Download QAT driver tarball + get_url: + url: "{{ qat_driver_url }}" + dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - hosts: kube-node become: yes @@ -33,21 +31,13 @@ include_vars: file: kud-vars.yml tasks: - - name: Create a destination for driver folder in the target's /tmp - file: - state: directory - path: "{{ item }}" - with_items: - - "{{ base_dest }}/quick-assist/{{ qat_package }}" - - name: Create QAT dest folder + - name: Create destination folder for QAT check script file: state: directory - path: "qat" - - name: Register QAT env variable - shell: "echo {{ QAT_ENABLED | default(False) }}" + path: "{{ base_dest }}/qat" - name: Create QAT check script copy: - dest: "qat/qat.sh" + dest: "{{ base_dest }}/qat/qat.sh" content: | #!/bin/bash qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \ @@ -59,15 +49,11 @@ else echo "True" fi - - name: Changing perm of "sh", adding "+x" - shell: "chmod +x qat.sh" - args: - chdir: "qat" - warn: False - - name: Run the script and re-evaluate the variable. - command: "./qat.sh" + mode: 0755 + - name: Run QAT check script and re-evaluate the variable + command: ./qat.sh args: - chdir: "qat" + chdir: "{{ base_dest }}/qat" register: output - debug: var: output.stdout_lines @@ -75,70 +61,68 @@ QAT_ENABLED: "{{ output.stdout }}" - debug: var: output - - name: Clean the script and folder. + - name: Clean QAT check script and folder file: - path: qat + path: "{{ base_dest }}/qat" state: absent - - name: bootstrap | install qat compilation packages - package: - name: "{{ item }}" - state: present - with_items: - - pciutils - - build-essential - - libudev-dev - - pkg-config - when: QAT_ENABLED - - copy: - src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - dest: "{{ base_dest }}/quick-assist" - remote_src: no - when: QAT_ENABLED - - name: Extract QAT source code - unarchive: - src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" - dest: "{{ base_dest }}/quick-assist/{{ qat_package }}" - when: QAT_ENABLED - - name: Configure the target - command: ./configure --enable-icp-sriov=host - args: - chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" - when: QAT_ENABLED - - name: build qat driver - make: - chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" - target: "{{ item }}" - loop: - - clean - - uninstall - - install - when: QAT_ENABLED - - name: Create QAT driver folder in the target destination - file: - state: directory - path: "{{ item }}" - with_items: - - qat_driver_dest - when: QAT_ENABLED - - name: Copy QAT build directory qat target destination - command: "cp -r {{ base_dest }}/quick-assist/{{ qat_package }}/build/ /root/qat_driver_dest/" - when: QAT_ENABLED - - name: Copy QAT driver install script to target folder - command: "cp {{ playbook_dir }}/install_qat.sh /root/qat_driver_dest/build/install.sh" - when: QAT_ENABLED - - name: Copy QAT to target folder - command: "cp /etc/default/qat /root/qat_driver_dest/build" - when: QAT_ENABLED - - name: Changing perm of "install.sh", adding "+x" - file: dest=~/qat_driver_dest/build/install.sh mode=a+x - when: QAT_ENABLED - - name: Run a script with arguments - command: ./install.sh chdir=/root/qat_driver_dest/build - when: QAT_ENABLED - - name: get qat devices - shell: /usr/local/bin/adf_ctl status | grep up | awk '{print $4 substr($1, 4)}' | tr -d ',' - register: qat_devices - when: QAT_ENABLED - - name: Updating the qat device SSL values to avoid duplication - command: "./substitute.sh chdir={{ playbook_dir }}" + - name: Install QAT driver + block: + - name: Install QAT compilation packages + package: + name: "{{ item }}" + state: present + with_items: + - pciutils + - build-essential + - libudev-dev + - pkg-config + - name: Create destination folder for QAT source code + file: + state: directory + path: "{{ qat_dest }}/{{ qat_package }}" + - name: Extract QAT source code + unarchive: + src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + dest: "{{ qat_dest }}/{{ qat_package }}" + - name: Configure the target + command: ./configure --enable-icp-sriov=host + args: + chdir: "{{ qat_dest }}/{{ qat_package }}" + - name: Build QAT driver + make: + chdir: "{{ qat_dest }}/{{ qat_package }}" + target: "{{ item }}" + loop: + - clean + - uninstall + - install + - name: Copy QAT driver install script to target folder + copy: + src: "install_qat.sh" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + mode: 0755 + - name: Copy /etc/default/qat to target folder + copy: + src: "/etc/default/qat" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + remote_src: yes + - name: Run a script with arguments + command: ./install_qat.sh + args: + chdir: "{{ qat_dest }}/{{ qat_package }}/build" + - name: Copy QAT substitue script to target folder + copy: + src: "substitute.sh" + dest: "{{ qat_dest }}/{{ qat_package }}/build" + mode: 0755 + - name: Update the QAT device SSL values to avoid duplication + command: ./substitute.sh + args: + chdir: "{{ qat_dest }}/{{ qat_package }}/build" + - name: Restart acceleration driver framework + command: adf_ctl restart + - name: Restart QAT service + service: + name: qat_service + state: restarted when: QAT_ENABLED diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 226f4568..db6b224c 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -122,7 +122,7 @@ function install_addons { tee $cluster_log/setup-kud.log # The order of KUD_ADDONS is important: some plugins (sriov, qat) # require nfd to be enabled. - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-${addon}.yml | \ @@ -131,7 +131,7 @@ function install_addons { echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do pushd $kud_tests bash ${addon}.sh case $addon in diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh index 2f8d212a..8365f700 100755 --- a/kud/tests/qat.sh +++ b/kud/tests/qat.sh @@ -10,16 +10,13 @@ set -o pipefail -qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \ - do lspci -d 8086:$i -m; done |\ - grep -i "Quick*" | head -n 1 | cut -d " " -f 5 ) -#Checking if the QAT device is on the node -if [ -z "$qat_device" ]; then - echo "False. This test case cannot run. Qat device unavailable." +qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity."qat.intel.com/cy2_dc2">="1") | .metadata.name') +if [ -z "$qat_capable_nodes" ]; then + echo "This test case cannot run. QAT device unavailable." QAT_ENABLED=False exit 0 else - echo "True. Can run QAT on this device." + echo "Can run QAT on this cluster." QAT_ENABLED=True fi @@ -78,9 +75,7 @@ kubectl create -f $HOME/$pod_name.yaml --validate=false allocated_node_resource=$(kubectl describe node | grep "qat.intel.com" | tail -n1 |awk '{print $(NF)}') echo "The allocated resource of the node is: " $allocated_node_resource -adf_ctl restart -systemctl restart qat_service -kubectl exec -it pod-case-01 -- openssl engine -c -t qat +kubectl exec pod-case-01 -- openssl engine -c -t qat kubectl delete pod $pod_name --now echo "Test complete." |