diff options
Diffstat (limited to 'kud/deployment_infra')
| -rw-r--r-- | kud/deployment_infra/images/qat_plugin_privileges.yaml | 40 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-qat.yml | 15 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/install_qat.sh | 238 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/kud-vars.yml | 12 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/preconfigure-qat.yml | 144 | ||||
| -rwxr-xr-x | kud/deployment_infra/playbooks/substitute.sh | 23 |
6 files changed, 466 insertions, 6 deletions
diff --git a/kud/deployment_infra/images/qat_plugin_privileges.yaml b/kud/deployment_infra/images/qat_plugin_privileges.yaml new file mode 100644 index 00000000..af98f367 --- /dev/null +++ b/kud/deployment_infra/images/qat_plugin_privileges.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-qat-kernel-plugin + labels: + app: intel-qat-kernel-plugin +spec: + selector: + matchLabels: + app: intel-qat-kernel-plugin + template: + metadata: + labels: + app: intel-qat-kernel-plugin + spec: + containers: + - name: intel-qat-kernel-plugin + securityContext: + privileged: true + image: akhilak/intel-qat-plugin:0.15.0 + imagePullPolicy: IfNotPresent + command: ["/usr/local/bin/intel_qat_device_plugin", "-mode", "kernel"] + volumeMounts: + - name: devfs + mountPath: /dev + - name: etcdir + mountPath: /etc + readOnly: true + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: etcdir + hostPath: + path: /etc + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins + - name: devfs + hostPath: + path: /dev diff --git a/kud/deployment_infra/playbooks/configure-qat.yml b/kud/deployment_infra/playbooks/configure-qat.yml new file mode 100644 index 00000000..1225b3d4 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-qat.yml @@ -0,0 +1,15 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- import_playbook: preconfigure-qat.yml +- hosts: localhost + tasks: + - name: Apply QAT plugin previleges Daemonset + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/qat_plugin_privileges.yaml" diff --git a/kud/deployment_infra/playbooks/install_qat.sh b/kud/deployment_infra/playbooks/install_qat.sh new file mode 100644 index 00000000..57adb923 --- /dev/null +++ b/kud/deployment_infra/playbooks/install_qat.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +# Precondition: +# QAT device installed, such as lspci | grep 37c8 +# Enable grub with "intel_iommu=on iommu=pt" + +ROOT= + +MV=mv +RM=rm +ECHO=echo +SLEEP=sleep +INSTALL="/usr/bin/install -c" +MKDIR_P="mkdir -p" +libdir=/usr/local/lib +bindir=/usr/local/bin + +am__append_1="drivers/crypto/qat/qat_dh895xcc/qat_dh895xcc.ko\ + drivers/crypto/qat/qat_dh895xccvf/qat_dh895xccvf.ko" + +am__append_2="qat_895xcc.bin qat_895xcc_mmp.bin" +am__append_3="dh895xcc_dev0.conf dh895xcc_dev1.conf dh895xccvf_dev0.conf.vm" + +# Kernel modules list +KO_MODULES_LIST="drivers/crypto/qat/qat_common/intel_qat.ko \ + drivers/crypto/qat/qat_c62x/qat_c62x.ko \ + drivers/crypto/qat/qat_c62xvf/qat_c62xvf.ko \ + drivers/crypto/qat/qat_d15xx/qat_d15xx.ko \ + drivers/crypto/qat/qat_d15xxvf/qat_d15xxvf.ko \ + drivers/crypto/qat/qat_c3xxx/qat_c3xxx.ko \ + drivers/crypto/qat/qat_c3xxxvf/qat_c3xxxvf.ko $am__append_1" + +# Firmwares list +BIN_LIST="qat_c3xxx.bin qat_c3xxx_mmp.bin qat_c62x.bin \ + qat_c62x_mmp.bin qat_mmp.bin qat_d15xx.bin qat_d15xx_mmp.bin \ + $am__append_2" +CONFIG_LIST="c3xxx_dev0.conf \ + c3xxxvf_dev0.conf.vm \ + c6xx_dev0.conf \ + c6xx_dev1.conf \ + c6xx_dev2.conf \ + c6xxvf_dev0.conf.vm \ + d15xx_dev0.conf \ + d15xxpf_dev0.conf \ + d15xxvf_dev0.conf.vm \ + $am__append_3" + +QAT_DH895XCC_NUM_VFS=32 +QAT_DHC62X_NUM_VFS=16 +QAT_DHD15XX_NUM_VFS=16 +QAT_DHC3XXX_NUM_VFS=16 + +# Device information variables +INTEL_VENDORID="8086" +DH895_DEVICE_NUMBER="0435" +DH895_DEVICE_NUMBER_VM="0443" +C62X_DEVICE_NUMBER="37c8" +C62X_DEVICE_NUMBER_VM="37c9" +D15XX_DEVICE_NUMBER="6f54" +D15XX_DEVICE_NUMBER_VM="6f55" +C3XXX_DEVICE_NUMBER="19e2" +C3XXX_DEVICE_NUMBER_VM="19e3" +numC62xDevice=`lspci -vnd 8086: | egrep -c "37c8|37c9"` +numD15xxDevice=`lspci -vnd 8086: | egrep -c "6f54|6f55"` +numDh895xDevice=`lspci -vnd 8086: | egrep -c "0435|0443"` +numC3xxxDevice=`lspci -vnd 8086: | egrep -c "19e2|19e3"` +numDh895xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER"` +numDh895xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$DH895_DEVICE_NUMBER_VM"` +numC62xDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER"` +numD15xxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER"` +numC3xxxDevicesP=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER"` +MODPROBE_BLACKLIST_FILE="blacklist-qat-vfs.conf" + +# load vfio-pci +$ECHO "Loading module vfio-pci" +modprobe vfio-pci + +# qat-driver +$ECHO "Installing driver in `\uname -r`" +INSTALL_MOD_DIR=/lib/modules/`\uname -r`/updates/ +for ko in $KO_MODULES_LIST; do + base=${ko%/*}; + file=${ko##*/}; + mkdir -p $ROOT$INSTALL_MOD_DIR$base + $INSTALL $file $ROOT$INSTALL_MOD_DIR$base +done + +# qat-adf-ctl +if [ ! -d $ROOT$bindir ]; then + $MKDIR_P $ROOT$bindir; +fi; +$INSTALL -D -m 750 adf_ctl $ROOT$bindir/adf_ctl; + +# qat-service +if [ ! -d $ROOT/lib/firmware/qat_fw_backup ]; then + $MKDIR_P $ROOT/lib/firmware/qat_fw_backup; +fi; + +for bin in $BIN_LIST; do + if [ -e $ROOT/lib/firmware/$bin ]; then + mv $ROOT/lib/firmware/$bin $ROOT/lib/firmware/qat_fw_backup/$bin; + fi; + if [ -e $bin ]; then + $INSTALL -D -m 750 $bin $ROOT/lib/firmware/$bin; + fi; +done; + +if [ ! -d $ROOT/etc/qat_conf_backup ]; then + $MKDIR_P $ROOT/etc/qat_conf_backup; +fi; +$MV $ROOT/etc/dh895xcc*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/c6xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/d15xx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; +$MV $ROOT/etc/c3xxx*.conf $ROOT/etc/qat_conf_backup/ 2>/dev/null; + +for ((dev=0; dev<$numDh895xDevicesP; dev++)); do + $INSTALL -D -m 640 dh895xcc_dev0.conf $ROOT/etc/dh895xcc_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DH895XCC_NUM_VFS; vf_dev++)); do + vf_dev_num=$(($dev * $QAT_DH895XCC_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 dh895xccvf_dev0.conf.vm $ROOT/etc/dh895xccvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numC62xDevicesP; dev++)); do + $INSTALL -D -m 640 c6xx_dev$(($dev%3)).conf $ROOT/etc/c6xx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHC62X_NUM_VFS; vf_dev++)); do + vf_dev_num=$(($dev * $QAT_DHC62X_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 c6xxvf_dev0.conf.vm $ROOT/etc/c6xxvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numD15xxDevicesP; dev++)); do + $INSTALL -D -m 640 d15xx_dev$(($dev%3)).conf $ROOT/etc/d15xx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHD15XX_NUM_VFS; vf_dev++)); do + vf_dev_num=$(($dev * $QAT_DHD15XX_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 d15xxvf_dev0.conf.vm $ROOT/etc/d15xxvf_dev$vf_dev_num.conf; + done; +done; + +for ((dev=0; dev<$numC3xxxDevicesP; dev++)); do + $INSTALL -D -m 640 c3xxx_dev0.conf $ROOT/etc/c3xxx_dev$dev.conf; + for ((vf_dev = 0; vf_dev<$QAT_DHC3XXX_NUM_VFS; vf_dev++)); do + vf_dev_num=$(($dev * $QAT_DHC3XXX_NUM_VFS + $vf_dev)); + $INSTALL -D -m 640 c3xxxvf_dev0.conf.vm $ROOT/etc/c3xxxvf_dev$vf_dev_num.conf; + done; +done; + +$ECHO "Creating startup and kill scripts"; +if [ ! -d $ROOT/etc/modprobe.d ]; then + $MKDIR_P $ROOT/etc/modprobe.d; +fi; +$INSTALL -D -m 750 qat_service $ROOT/etc/init.d/qat_service; +$INSTALL -D -m 750 qat_service_vfs $ROOT/etc/init.d/qat_service_vfs; +$INSTALL -D -m 750 qat $ROOT/etc/default/qat; +if [ -e $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE ] ; then + $RM $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; + +if [ $numDh895xDevicesP != 0 ];then + $ECHO "blacklist qat_dh895xccvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numC3xxxDevicesP != 0 ];then + $ECHO "blacklist qat_c3xxxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numC62xDevicesP != 0 ];then + $ECHO "blacklist qat_c62xvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; +if [ $numD15xxDevicesP != 0 ];then + $ECHO "blacklist qat_d15xxvf" >> $ROOT/etc/modprobe.d/$MODPROBE_BLACKLIST_FILE; +fi; + +if [ ! -d $ROOT$libdir ]; then + $MKDIR_P $ROOT$libdir; +fi; +if [ ! -d $ROOT/etc/ld.so.conf.d ]; then + $MKDIR_P $ROOT/etc/ld.so.conf.d; +fi; +if [ ! -d $ROOT/lib/modules/`\uname -r`/kernel/drivers ]; then + $MKDIR_P $ROOT/lib/modules/`\uname -r`/kernel/drivers; +fi; +if [ ! -d $ROOT/etc/udev/rules.d ]; then + $MKDIR_P $ROOT/etc/udev/rules.d; +fi; + +$ECHO "Copying libqat_s.so to $ROOT$libdir"; +$INSTALL -D -m 755 libqat_s.so $ROOT$libdir/libqat_s.so; +$ECHO "Copying libusdm_drv_s.so to $ROOT$libdir"; +$INSTALL -D -m 755 libusdm_drv_s.so $ROOT$libdir/libusdm_drv_s.so; +$ECHO $libdir > $ROOT/etc/ld.so.conf.d/qat.conf; ldconfig; + +$ECHO "Copying usdm module to system drivers"; +$INSTALL usdm_drv.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers"; +$INSTALL qat_api.ko "$ROOT/lib/modules/`\uname -r`/kernel/drivers"; +$ECHO "Creating udev rules"; +if [ ! -e $ROOT/etc/udev/rules.d/00-qat.rules ]; then + echo 'KERNEL=="qat_adf_ctl" MODE="0660" GROUP="qat"' > $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="qat_dev_processes" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="usdm_drv" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="uio*" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; + echo 'KERNEL=="hugepages" MODE="0660" GROUP="qat"' >> $ROOT/etc/udev/rules.d/00-qat.rules; +fi; +$ECHO "Creating module.dep file for QAT released kernel object"; +$ECHO "This will take a few moments"; +depmod -a; +if [ `lsmod | grep "usdm_drv" | wc -l` != "0" ]; then + $ECHO "rmmod usdm_drv"; + rmmod usdm_drv; +fi; +if [ -e /sbin/chkconfig ] ; then + chkconfig --add qat_service; +elif [ -e /usr/sbin/update-rc.d ]; then + $ECHO "update-rc.d qat_service defaults"; + update-rc.d qat_service defaults; +fi; + +$ECHO "Starting QAT service"; +/etc/init.d/qat_service shutdown; +$SLEEP 3; +/etc/init.d/qat_service start; +/etc/init.d/qat_service_vfs start; + +# load kernel vf module for QAT device plugin +numC62xDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C62X_DEVICE_NUMBER_VM"` +numD15xxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$D15XX_DEVICE_NUMBER_VM"` +numC3xxxDevicesV=`lspci -n | egrep -c "$INTEL_VENDORID:$C3XXX_DEVICE_NUMBER_VM"` +if [ $numC62xDevicesV != 0 ];then + $ECHO "Loading qat_c62xvf"; + modprobe qat_c62xvf +fi +if [ $numC3xxxDevicesV != 0 ];then + $ECHO "Loading qat_c3xxxvf"; + modprobe qat_c3xxxvf +fi +if [ $numD15xxDevicesV != 0 ];then + $ECHO "Loading qat_d15xxvf"; + modprobe qat_d15xxvf +fi + diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 2a25049a..a592a909 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -39,14 +39,14 @@ istio_source_type: "tarball" istio_version: 1.0.3 istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz" -sriov_dest: "{{ base_dest }}/sriov" -driver_source_type: "tarball" -driver_version: 3.7.34 -driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ driver_version }}.tar.gz" -package: iavf-3.7.34 - go_version: '1.12.5' kubespray_version: 2.10.4 helm_client_version: 2.13.1 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634 ansible_version: 2.7.10 + +qat_dest: "{{ base_dest }}/qat" +qat_driver_source_type: "tarball" +qat_driver_version: 1.7.l.4.6.0-00025 +qat_driver_url: "https://01.org/sites/default/files/downloads/{{ qat_package }}.tar.gz" +qat_package: qat1.7.l.4.6.0-00025 diff --git a/kud/deployment_infra/playbooks/preconfigure-qat.yml b/kud/deployment_infra/playbooks/preconfigure-qat.yml new file mode 100644 index 00000000..f5d797f1 --- /dev/null +++ b/kud/deployment_infra/playbooks/preconfigure-qat.yml @@ -0,0 +1,144 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: Create QAT dest folder + file: + state: directory + path: "{{ qat_dest }}" + - name: Fetching QAT driver + block: + - name: Download QAT driver tarball + get_url: + url: "{{ qat_driver_url }}" + dest: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + +- hosts: kube-node + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: Create a destination for driver folder in the target's /tmp + file: + state: directory + path: "{{ item }}" + with_items: + - "{{ base_dest }}/quick-assist/{{ qat_package }}" + - name: Create QAT dest folder + file: + state: directory + path: "qat" + - name: Register QAT env variable + shell: "echo {{ QAT_ENABLED | default(False) }}" + - name: Create QAT check script + copy: + dest: "qat/qat.sh" + content: | + #!/bin/bash + qat_device=$( for i in 0434 0435 37c8 6f54 19e2; \ + do lspci -d 8086:$i -m; done |\ + grep -i "Quick*" | head -n 1 | cut -d " " -f 5 ) + if [ -z "$qat_device" ]; then + echo "False" + exit 0 + else + echo "True" + fi + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x qat.sh" + args: + chdir: "qat" + warn: False + - name: Run the script and re-evaluate the variable. + command: "./qat.sh" + args: + chdir: "qat" + register: output + - debug: + var: output.stdout_lines + - set_fact: + QAT_ENABLED: "{{ output.stdout }}" + - debug: + var: output + - name: Clean the script and folder. + file: + path: qat + state: absent + - name: bootstrap | install qat compilation packages + package: + name: "{{ item }}" + state: present + with_items: + - pciutils + - build-essential + - libudev-dev + - pkg-config + when: QAT_ENABLED + - copy: + src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + dest: "{{ base_dest }}/quick-assist" + remote_src: no + when: QAT_ENABLED + - name: Extract QAT source code + unarchive: + src: "{{ qat_dest }}/{{ qat_package }}.tar.gz" + dest: "{{ base_dest }}/quick-assist/{{ qat_package }}" + when: QAT_ENABLED + - name: Configure the target + command: ./configure --enable-icp-sriov=host + args: + chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" + when: QAT_ENABLED + - name: build qat driver + make: + chdir: "{{ base_dest }}/quick-assist/{{ qat_package }}" + target: "{{ item }}" + loop: + - clean + - uninstall + - install + when: QAT_ENABLED + - name: Create QAT driver folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - qat_driver_dest + when: QAT_ENABLED + - name: Copy QAT build directory qat target destination + command: "cp -r {{ base_dest }}/quick-assist/{{ qat_package }}/build/ /root/qat_driver_dest/" + when: QAT_ENABLED + - name: Copy QAT driver install script to target folder + command: "cp {{ playbook_dir }}/install_qat.sh /root/qat_driver_dest/build/install.sh" + when: QAT_ENABLED + - name: Copy QAT to target folder + command: "cp /etc/default/qat /root/qat_driver_dest/build" + when: QAT_ENABLED + - name: Changing perm of "install.sh", adding "+x" + file: dest=~/qat_driver_dest/build/install.sh mode=a+x + when: QAT_ENABLED + - name: Run a script with arguments + command: ./install.sh chdir=/root/qat_driver_dest/build + when: QAT_ENABLED + - name: get qat devices + shell: /usr/local/bin/adf_ctl status | grep up | awk '{print $4 substr($1, 4)}' | tr -d ',' + register: qat_devices + when: QAT_ENABLED + - name: Updating the qat device SSL values to avoid duplication + command: "./substitute.sh chdir={{ playbook_dir }}" + when: QAT_ENABLED diff --git a/kud/deployment_infra/playbooks/substitute.sh b/kud/deployment_infra/playbooks/substitute.sh new file mode 100755 index 00000000..f6907a42 --- /dev/null +++ b/kud/deployment_infra/playbooks/substitute.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +for file in $(find /etc/*.conf -type f -name "c6xxvf_dev*.conf"); do + device_id=$( echo $file | cut -d '_' -f 2 | tr -cd '[[:digit:]]') + echo $device_id + cat /etc/c6xxvf_dev${device_id}.conf + sed -i "s/\[SSL\]/\[SSL${device_id}\]/g" /etc/c6xxvf_dev${device_id}.conf +done + +for file in $(find /etc/*.conf -type f -name "c6xx_dev*.conf"); do + dev_id=$( echo $file | cut -d '_' -f 2 | tr -cd '[[:digit:]]') + echo $dev_id + cat /etc/c6xx_dev${dev_id}.conf + sed -i "s/\[SSL\]/\[SSL${dev_id}\]/g" /etc/c6xx_dev${dev_id}.conf +done |