diff options
12 files changed, 972 insertions, 2 deletions
diff --git a/kud/deployment_infra/helm/qat-device-plugin/.helmignore b/kud/deployment_infra/helm/qat-device-plugin/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kud/deployment_infra/helm/qat-device-plugin/Chart.yaml b/kud/deployment_infra/helm/qat-device-plugin/Chart.yaml new file mode 100644 index 00000000..1697abb0 --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/Chart.yaml @@ -0,0 +1,25 @@ +# Copyright 2021 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +appVersion: 0.19.0-kerneldrv +description: | + A device plugin that provides support for Intel QuickAssist + Technology (QAT) devices under Kubernetes. +home: https://github.com/intel/intel-device-plugins-for-kubernetes +name: qat-device-plugin +sources: + - https://github.com/intel/intel-device-plugins-for-kubernetes +type: application +version: 0.1.0 diff --git a/kud/deployment_infra/helm/qat-device-plugin/templates/_helpers.tpl b/kud/deployment_infra/helm/qat-device-plugin/templates/_helpers.tpl new file mode 100644 index 00000000..77889d5d --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/templates/_helpers.tpl @@ -0,0 +1,52 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "qat-device-plugin.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "qat-device-plugin.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "qat-device-plugin.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "qat-device-plugin.labels" -}} +helm.sh/chart: {{ include "qat-device-plugin.chart" . }} +{{ include "qat-device-plugin.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "qat-device-plugin.selectorLabels" -}} +app.kubernetes.io/name: {{ include "qat-device-plugin.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/kud/deployment_infra/helm/qat-device-plugin/templates/config.yaml b/kud/deployment_infra/helm/qat-device-plugin/templates/config.yaml new file mode 100644 index 00000000..24ffaa4a --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/templates/config.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.config.name }} + labels: + {{- include "qat-device-plugin.labels" . | nindent 4 }} +data: + VERBOSITY: {{ .Values.config.logLevel | quote }} diff --git a/kud/deployment_infra/helm/qat-device-plugin/templates/daemonset.yaml b/kud/deployment_infra/helm/qat-device-plugin/templates/daemonset.yaml new file mode 100644 index 00000000..c94ff330 --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/templates/daemonset.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "qat-device-plugin.fullname" . }} + labels: + {{- include "qat-device-plugin.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "qat-device-plugin.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "qat-device-plugin.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + containers: + - name: intel-qat-kernel-plugin + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 10 }} + env: + - name: VERBOSITY + valueFrom: + configMapKeyRef: + name: {{ .Values.config.name }} + key: VERBOSITY + args: ["-mode", "kernel", "-v", "$(VERBOSITY)"] + volumeMounts: + - name: devfs + mountPath: /dev + - name: etcdir + mountPath: /etc + readOnly: true + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: etcdir + hostPath: + path: /etc + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins + - name: devfs + hostPath: + path: /dev diff --git a/kud/deployment_infra/helm/qat-device-plugin/templates/drivers/daemonset.yaml b/kud/deployment_infra/helm/qat-device-plugin/templates/drivers/daemonset.yaml new file mode 100644 index 00000000..7ba04047 --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/templates/drivers/daemonset.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "qat-device-plugin.fullname" . }}-qat-driver-installer + labels: + {{- include "qat-device-plugin.labels" . | nindent 4 }} + role: qat-driver-installer +spec: + selector: + matchLabels: + {{- include "qat-device-plugin.selectorLabels" . | nindent 6 }} + role: qat-driver-installer + template: + metadata: + labels: + {{- include "qat-device-plugin.selectorLabels" . | nindent 8 }} + role: qat-driver-installer + spec: + hostPID: true + {{- with .Values.qatDriver.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.qatDriver.podSecurityContext | nindent 8 }} + initContainers: + - image: "{{ .Values.qatDriver.image.repository }}:{{ .Values.qatDriver.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.qatDriver.image.pullPolicy }} + name: qat-driver-installer + securityContext: + {{- toYaml .Values.qatDriver.securityContext | nindent 12 }} + resources: + {{- toYaml .Values.qatDriver.resources | nindent 12 }} + volumeMounts: + - name: qat-install-dir + mountPath: "/usr/local/qat" + - name: root-dir + mountPath: "/root" + - name: lib-modules-dir + mountPath: "/root/lib/modules" + - name: run-systemd-dir + mountPath: "/root/run/systemd/system" + containers: + - image: "gcr.io/google-containers/pause:3.2" + name: pause + volumes: + - name: qat-install-dir + hostPath: + path: "/opt/qat" + - name: root-dir + hostPath: + path: "/" + - name: lib-modules-dir + hostPath: + path: "/lib/modules" + - name: run-systemd-dir + hostPath: + path: "/run/systemd/system" + {{- with .Values.qatDriver.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.qatDriver.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.qatDriver.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/kud/deployment_infra/helm/qat-device-plugin/values.yaml b/kud/deployment_infra/helm/qat-device-plugin/values.yaml new file mode 100644 index 00000000..459c36b1 --- /dev/null +++ b/kud/deployment_infra/helm/qat-device-plugin/values.yaml @@ -0,0 +1,49 @@ +config: + name: intel-qat-plugin-config + + # logLevel sets the plugin's log level. + logLevel: 4 + +imagePullSecrets: [] + +image: + repository: integratedcloudnative/intel-qat-plugin + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +nodeSelector: + feature.node.kubernetes.io/iommu-enabled: "true" + feature.node.kubernetes.io/custom-qat: "true" + +securityContext: + readOnlyRootFilesystem: true + privileged: true + +resources: {} + +affinity: {} + +tolerations: {} + +qatDriver: + image: + repository: integratedcloudnative/qat-driver-installer + pullPolicy: IfNotPresent + tag: latest + imagePullSecrets: [] + + podSecurityContext: {} + # fsGroup: 2000 + + securityContext: + privileged: true + + nodeSelector: + feature.node.kubernetes.io/iommu-enabled: "true" + feature.node.kubernetes.io/custom-qat: "true" + + resources: {} + + affinity: {} + + tolerations: {} diff --git a/kud/deployment_infra/installers/Dockerfile.qat-driver-installer b/kud/deployment_infra/installers/Dockerfile.qat-driver-installer new file mode 100644 index 00000000..7d885a59 --- /dev/null +++ b/kud/deployment_infra/installers/Dockerfile.qat-driver-installer @@ -0,0 +1,21 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y \ + bc \ + build-essential \ + curl \ + kmod \ + libelf-dev \ + libssl-dev \ + libudev-dev \ + pciutils \ + pkg-config \ + && \ + rm -rf /var/lib/apt/lists/* + +COPY _common.sh / +COPY _qat-driver-installer.sh / +COPY entrypoint-qat-driver-installer.sh /entrypoint.sh + +CMD /entrypoint.sh diff --git a/kud/deployment_infra/installers/Makefile b/kud/deployment_infra/installers/Makefile index 17c651dd..99aadbc0 100644 --- a/kud/deployment_infra/installers/Makefile +++ b/kud/deployment_infra/installers/Makefile @@ -1,6 +1,6 @@ REGISTRY?=integratedcloudnative TAG?=latest -IMAGES=iavf-driver-installer +IMAGES=iavf-driver-installer qat-driver-installer .PHONY: all $(IMAGES) diff --git a/kud/deployment_infra/installers/_qat-driver-installer.sh b/kud/deployment_infra/installers/_qat-driver-installer.sh new file mode 100644 index 00000000..5ecc2f5a --- /dev/null +++ b/kud/deployment_infra/installers/_qat-driver-installer.sh @@ -0,0 +1,514 @@ +#!/bin/bash +# +# The functions below are captured from the Makefile targets. They +# cannot be run in a container as-is due to absolute paths, so they +# are recreated here. +# +# Note also that the portions of qat-driver-install that deal with +# rc.d are removed: they are intended to be handled by the deployed +# DaemonSet. The rest is contained in _qat_service_start. +# +# The checks for loaded modules are moved to _qat_check_started. + +BIN_LIST="qat_c3xxx.bin qat_c3xxx_mmp.bin qat_c62x.bin \ + qat_c62x_mmp.bin qat_mmp.bin qat_d15xx.bin qat_d15xx_mmp.bin \ + qat_200xx.bin qat_200xx_mmp.bin qat_895xcc.bin qat_895xcc_mmp.bin" + +numDh895xDevicesP=$(lspci -n | grep -c "8086:0435") || true +numDh895xDevicesV=$(lspci -n | grep -c "8086:0443") || true +numC62xDevicesP=$(lspci -n | grep -c "8086:37c8") || true +numC62xDevicesV=$(lspci -n | grep -c "8086:37c9") || true +numD15xxDevicesP=$(lspci -n | grep -c "8086:6f54") || true +numD15xxDevicesV=$(lspci -n | grep -c "8086:6f55") || true +numC3xxxDevicesP=$(lspci -n | grep -c "8086:19e2") || true +numC3xxxDevicesV=$(lspci -n | grep -c "8086:19e3") || true +num200xxDevicesP=$(lspci -n | grep -c "8086:18ee") || true +num200xxDevicesV=$(lspci -n | grep -c "8086:18ef") || true + +_qat_driver_install() { + info "Installing drivers" + if [[ -z "${KERNEL_MOD_SIGN_CMD}" ]]; then + info "No driver signing required" + INSTALL_MOD_PATH=${ROOT_MOUNT_DIR} make KDIR="${KERNEL_SRC_DIR}" -C "${QAT_INSTALL_DIR_CONTAINER}/quickassist/qat" mod_sign_cmd=":" modules_install + else + info "Driver signing is required" + INSTALL_MOD_PATH=${ROOT_MOUNT_DIR} make KDIR="${KERNEL_SRC_DIR}" -C "${QAT_INSTALL_DIR_CONTAINER}/quickassist/qat" mod_sign_cmd="${KERNEL_MOD_SIGN_CMD}" modules_install + fi +} + +_adf_ctl_install() { + info "Installing adf_ctl" + install -D -m 750 "${QAT_INSTALL_DIR_CONTAINER}/quickassist/utilities/adf_ctl/adf_ctl" "${ROOT_MOUNT_DIR}/usr/local/bin/adf_ctl" +} + +_adf_ctl_uninstall() { + info "Uninstalling adf_ctl" + # rm ${ROOT_MOUNT_DIR}/usr/local/bin/adf_ctl + return 0 +} + +_rename_ssl_conf_section() { + info "Renaming SSL section in conf files" + restore_nullglob=$(shopt -p | grep nullglob) + shopt -s nullglob + for file in ${ROOT_MOUNT_DIR}/etc/dh895xcc_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/dh895xcc_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/c6xx_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/c6xx_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/d15xx_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/d15xx_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/c3xxx_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/c3xxx_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/200xx_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/200xx_dev${dev}.conf" + done + + for file in ${ROOT_MOUNT_DIR}/etc/dh895xccvf_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/dh895xccvf_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/c6xxvf_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/c6xxvf_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/d15xxvf_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/d15xxvf_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/c3xxxvf_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/c3xxxvf_dev${dev}.conf" + done + for file in ${ROOT_MOUNT_DIR}/etc/200xxvf_dev*.conf; do + dev=$(echo "$file" | cut -d '_' -f 2 | tr -cd '[:digit:]') + sed -i "s/\[SSL\]/\[SSL${dev}\]/g" "${ROOT_MOUNT_DIR}/etc/200xxvf_dev${dev}.conf" + done + $restore_nullglob +} + +_qat_service_install() { + local -r QAT_DH895XCC_NUM_VFS=32 + local -r QAT_DHC62X_NUM_VFS=16 + local -r QAT_DHD15XX_NUM_VFS=16 + local -r QAT_DHC3XXX_NUM_VFS=16 + local -r QAT_DH200XX_NUM_VFS=16 + local -r DEVICES="0435 0443 37c8 37c9 6f54 6f55 19e2 19e3 18ee 18ef" + + info "Installing service" + pushd "${QAT_INSTALL_DIR_CONTAINER}/build" > /dev/null + + if [[ ! -d ${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup ]]; then + mkdir -p "${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup" + fi + for bin in ${BIN_LIST}; do + if [[ -e ${ROOT_MOUNT_DIR}/lib/firmware/${bin} ]]; then + mv "${ROOT_MOUNT_DIR}/lib/firmware/${bin}" "${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup/${bin}" + fi + if [[ -e ${bin} ]]; then + install -D -m 750 "${bin}" "${ROOT_MOUNT_DIR}/lib/firmware/${bin}" + fi + done + if [[ ! -d ${ROOT_MOUNT_DIR}/etc/qat_conf_backup ]]; then + mkdir "${ROOT_MOUNT_DIR}/etc/qat_conf_backup" + fi + mv "${ROOT_MOUNT_DIR}/etc/dh895xcc*.conf" "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/" 2>/dev/null || true + mv "${ROOT_MOUNT_DIR}/etc/c6xx*.conf" "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/" 2>/dev/null || true + mv "${ROOT_MOUNT_DIR}/etc/d15xx*.conf" "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/" 2>/dev/null || true + mv "${ROOT_MOUNT_DIR}/etc/c3xxx*.conf" "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/" 2>/dev/null || true + mv "${ROOT_MOUNT_DIR}/etc/200xx*.conf" "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/" 2>/dev/null || true + if [[ "${QAT_ENABLE_SRIOV}" != "guest" ]]; then + for ((dev=0; dev<numDh895xDevicesP; dev++)); do + install -D -m 640 dh895xcc_dev0.conf "${ROOT_MOUNT_DIR}/etc/dh895xcc_dev${dev}.conf" + done + for ((dev=0; dev<numC62xDevicesP; dev++)); do + install -D -m 640 c6xx_dev$((dev%3)).conf "${ROOT_MOUNT_DIR}/etc/c6xx_dev${dev}.conf" + done + for ((dev=0; dev<numD15xxDevicesP; dev++)); do + install -D -m 640 d15xx_dev$((dev%3)).conf "${ROOT_MOUNT_DIR}/etc/d15xx_dev${dev}.conf" + done + for ((dev=0; dev<numC3xxxDevicesP; dev++)); do + install -D -m 640 c3xxx_dev0.conf "${ROOT_MOUNT_DIR}/etc/c3xxx_dev${dev}.conf" + done + for ((dev=0; dev<num200xxDevicesP; dev++)); do + install -D -m 640 200xx_dev0.conf "${ROOT_MOUNT_DIR}/etc/200xx_dev${dev}.conf" + done + fi + if [[ "${QAT_ENABLE_SRIOV}" == "host" ]]; then + for ((dev=0; dev<numDh895xDevicesP; dev++)); do + for ((vf_dev=0; vf_dev<QAT_DH895XCC_NUM_VFS; vf_dev++)); do + vf_dev_num=$((dev * QAT_DH895XCC_NUM_VFS + vf_dev)) + install -D -m 640 dh895xccvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/dh895xccvf_dev${vf_dev_num}.conf" + done + done + for ((dev=0; dev<numC62xDevicesP; dev++)); do + for ((vf_dev=0; vf_dev<QAT_DHC62X_NUM_VFS; vf_dev++)); do + vf_dev_num=$((dev * QAT_DHC62X_NUM_VFS + vf_dev)) + install -D -m 640 c6xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/c6xxvf_dev${vf_dev_num}.conf" + done + done + for ((dev=0; dev<numD15xxDevicesP; dev++)); do + for ((vf_dev=0; vf_dev<QAT_DHD15XX_NUM_VFS; vf_dev++)); do + vf_dev_num=$((dev * QAT_DHD15XX_NUM_VFS + vf_dev)) + install -D -m 640 d15xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/d15xxvf_dev${vf_dev_num}.conf" + done + done + for ((dev=0; dev<numC3xxxDevicesP; dev++)); do + for ((vf_dev=0; vf_dev<QAT_DHC3XXX_NUM_VFS; vf_dev++)); do + vf_dev_num=$((dev * QAT_DHC3XXX_NUM_VFS + vf_dev)) + install -D -m 640 c3xxxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/c3xxxvf_dev${vf_dev_num}.conf" + done + done + for ((dev=0; dev<num200xxDevicesP; dev++)); do + for ((vf_dev=0; vf_dev<QAT_DH200XX_NUM_VFS; vf_dev++)); do + vf_dev_num=$((dev * QAT_DH200XX_NUM_VFS + vf_dev)) + install -D -m 640 200xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/200xxvf_dev${vf_dev_num}.conf" + done + done + else + for ((dev=0; dev<numDh895xDevicesV; dev++)); do + install -D -m 640 dh895xccvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/dh895xccvf_dev${dev}.conf" + done + for ((dev=0; dev<numC62xDevicesV; dev++)); do + install -D -m 640 c6xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/c6xxvf_dev${dev}.conf" + done + for ((dev=0; dev<numD15xxDevicesV; dev++)); do + install -D -m 640 d15xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/d15xxvf_dev${dev}.conf" + done + for ((dev=0; dev<numC3xxxDevicesV; dev++)); do + install -D -m 640 c3xxxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/c3xxxvf_dev${dev}.conf" + done + for ((dev=0; dev<num200xxDevicesV; dev++)); do + install -D -m 640 200xxvf_dev0.conf.vm "${ROOT_MOUNT_DIR}/etc/200xxvf_dev${dev}.conf" + done + fi + _rename_ssl_conf_section + info "Creating startup and kill scripts" + install -D -m 750 qat_service "${ROOT_MOUNT_DIR}/etc/init.d/qat_service" + if [[ "${QAT_ENABLE_SRIOV}" == "host" ]]; then + install -D -m 750 qat_service_vfs "${ROOT_MOUNT_DIR}/etc/init.d/qat_service_vfs" + fi + if [[ "${QAT_ENABLE_SRIOV}" == "host" || "${QAT_ENABLE_SRIOV}" == "guest" ]]; then + echo "# Comment or remove next line to disable sriov" > "${ROOT_MOUNT_DIR}/etc/default/qat" + echo "SRIOV_ENABLE=1" >> "${ROOT_MOUNT_DIR}/etc/default/qat" + else + echo "# Remove comment on next line to enable sriov" > "${ROOT_MOUNT_DIR}/etc/default/qat" + echo "#SRIOV_ENABLE=1" >> "${ROOT_MOUNT_DIR}/etc/default/qat" + fi + echo "#LEGACY_LOADED=1" >> "${ROOT_MOUNT_DIR}/etc/default/qat" + rm -f "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + if [[ "${QAT_ENABLE_SRIOV}" == "host" ]]; then + if [[ ${numDh895xDevicesP} != 0 ]]; then + echo "blacklist qat_dh895xccvf" >> "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + fi + if [[ ${numC3xxxDevicesP} != 0 ]]; then + echo "blacklist qat_c3xxxvf" >> "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + fi + if [[ ${num200xxDevicesP} != 0 ]]; then + echo "blacklist qat_200xxvf" >> "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + fi + if [[ ${numC62xDevicesP} != 0 ]]; then + echo "blacklist qat_c62xvf" >> "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + fi + if [[ ${numD15xxDevicesP} != 0 ]]; then + echo "blacklist qat_d15xxvf" >> "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + fi + fi + echo "#ENABLE_KAPI=1" >> "${ROOT_MOUNT_DIR}/etc/default/qat" + info "Copying libqat_s.so to ${ROOT_MOUNT_DIR}/usr/local/lib" + install -D -m 755 libqat_s.so "${ROOT_MOUNT_DIR}/usr/local/lib/libqat_s.so" + info "Copying libusdm_drv_s.so to ${ROOT_MOUNT_DIR}/usr/local/lib" + install -D -m 755 libusdm_drv_s.so "${ROOT_MOUNT_DIR}/usr/local/lib/libusdm_drv_s.so" + echo /usr/local/lib > "${ROOT_MOUNT_DIR}/etc/ld.so.conf.d/qat.conf" + ldconfig -r "${ROOT_MOUNT_DIR}" + info "Copying usdm module to system drivers" + if [[ ! -z "${KERNEL_MOD_SIGN_CMD}" ]]; then + info "Need to sign driver usdm_drv.ko" + ${KERNEL_MOD_SIGN_CMD} usdm_drv.ko + info "Need to sign driver qat_api.ko" + ${KERNEL_MOD_SIGN_CMD} qat_api.ko + fi + install usdm_drv.ko "${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/kernel/drivers" + install qat_api.ko "${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/kernel/drivers" + if [[ ! $(chroot "${ROOT_MOUNT_DIR}" getent group qat) ]]; then + info "Creating qat group" + groupadd -R "${ROOT_MOUNT_DIR}" qat + else + info "Group qat already exists" + fi + info "Creating udev rules" + rm -f "${ROOT_MOUNT_DIR}/etc/udev/rules.d/00-qat.rules" + { + echo 'KERNEL=="qat_adf_ctl" MODE="0660" GROUP="qat"'; + echo 'KERNEL=="qat_dev_processes" MODE="0660" GROUP="qat"'; + echo 'KERNEL=="usdm_drv" MODE="0660" GROUP="qat"'; + echo 'ACTION=="add", DEVPATH=="/module/usdm_drv" SUBSYSTEM=="module" RUN+="/bin/mkdir /dev/hugepages/qat"'; + echo 'ACTION=="add", DEVPATH=="/module/usdm_drv" SUBSYSTEM=="module" RUN+="/bin/chgrp qat /dev/hugepages/qat"'; + echo 'ACTION=="add", DEVPATH=="/module/usdm_drv" SUBSYSTEM=="module" RUN+="/bin/chmod 0770 /dev/hugepages/qat"'; + echo 'ACTION=="remove", DEVPATH=="/module/usdm_drv" SUBSYSTEM=="module" RUN+="/bin/rmdir /dev/hugepages/qat"'; + for dev in ${DEVICES}; do + echo 'KERNEL=="uio*", ATTRS{vendor}=="0x'"$(echo "8086" | tr -d \")"'", ATTRS{device}=="0x'"$(echo "${dev}" | tr -d \")"'" MODE="0660" GROUP="qat"'; + done + } > "${ROOT_MOUNT_DIR}/etc/udev/rules.d/00-qat.rules" + info "Creating module.dep file for QAT released kernel object" + info "This will take a few moments" + depmod -a -b "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/depmod.d" + + popd > /dev/null +} + +_qat_service_start() { + if [[ $(lsmod | grep -c "usdm_drv") != "0" ]]; then + rmmod usdm_drv + fi + info "Starting QAT service" + info "... shutting down" + chroot "${ROOT_MOUNT_DIR}" /etc/init.d/qat_service shutdown || true + sleep 3 + info "... starting" + chroot "${ROOT_MOUNT_DIR}" /etc/init.d/qat_service start + if [[ "${QAT_ENABLE_SRIOV}" == "host" ]]; then + modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" vfio-pci + chroot "${ROOT_MOUNT_DIR}" /etc/init.d/qat_service_vfs start + fi + info "... started" +} + +_qat_check_started() { + if [[ $(lsmod | grep -c "usdm_drv") == "0" ]]; then + error "usdm_drv module not installed" + return "${RETCODE_ERROR}" + fi + if [[ ${numDh895xDevicesP} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_dh895xcc") == "0" ]]; then + error "qat_dh895xcc module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numC62xDevicesP} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_c62x") == "0" ]]; then + error "qat_c62x module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numD15xxDevicesP} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_d15xx") == "0" ]]; then + error "qat_d15xx module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numC3xxxDevicesP} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_c3xxx") == "0" ]]; then + error "qat_c3xxx module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${num200xxDevicesP} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_200xx") == "0" ]]; then + error "qat_200xx module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ "${QAT_ENABLE_SRIOV}" == "guest" ]]; then + if [[ ${numDh895xDevicesV} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_dh895xccvf") == "0" ]]; then + error "qat_dh895xccvf module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numC62xDevicesV} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_c62xvf") == "0" ]]; then + error "qat_c62xvf module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numD15xxDevicesV} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_d15xxvf") == "0" ]]; then + error "qat_d15xxvf module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${numC3xxxDevicesV} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_c3xxxvf") == "0" ]]; then + error "qat_c3xxxvf module not installed" + return "${RETCODE_ERROR}" + fi + fi + if [[ ${num200xxDevicesV} != 0 ]]; then + if [[ $(lsmod | grep -c "qat_200xxvf") == "0" ]]; then + error "qat_200xxvf module not installed" + return "${RETCODE_ERROR}" + fi + fi + fi + if [[ $("${ROOT_MOUNT_DIR}/usr/local/bin/adf_ctl" status | grep -c "state: down") != "0" ]]; then + error "QAT driver not activated" + return "${RETCODE_ERROR}" + fi +} + +_qat_service_shutdown() { + info "Stopping service" + if [[ $(lsmod | grep -c "qat") != "0" || -e ${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/updates/drivers/crypto/qat/qat_common/intel_qat.ko ]]; then + if [[ $(lsmod | grep -c "usdm_drv") != "0" ]]; then + rmmod usdm_drv + fi + if [[ -e ${ROOT_MOUNT_DIR}/etc/init.d/qat_service_upstream ]]; then + until chroot "${ROOT_MOUNT_DIR}" /etc/init.d/qat_service_upstream shutdown; do + sleep 1 + done + elif [[ -e ${ROOT_MOUNT_DIR}/etc/init.d/qat_service ]]; then + until chroot "${ROOT_MOUNT_DIR}" /etc/init.d/qat_service shutdown; do + sleep 1 + done + fi + fi +} + +_qat_service_uninstall() { + info "Uninstalling service" + if [[ $(lsmod | grep -c "qat") != "0" || -e ${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/updates/drivers/crypto/qat/qat_common/intel_qat.ko ]]; then + info "Removing the QAT firmware" + for bin in ${BIN_LIST}; do + if [[ -e ${ROOT_MOUNT_DIR}/lib/firmware/${bin} ]]; then + rm "${ROOT_MOUNT_DIR}/lib/firmware/${bin}" + fi + if [[ -e ${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup/${bin} ]]; then + mv "${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup/${bin}" "${ROOT_MOUNT_DIR}/lib/firmware/${bin}" + fi + done + + if [[ -d ${ROOT_MOUNT_DIR}/lib/firmware/qat_fw ]]; then + rm "${ROOT_MOUNT_DIR}/lib/firmware/qat_fw_backup" + fi + + if [[ -e ${ROOT_MOUNT_DIR}/etc/init.d/qat_service_upstream ]]; then + rm "${ROOT_MOUNT_DIR}/etc/init.d/qat_service_upstream" + rm "${ROOT_MOUNT_DIR}/usr/local/bin/adf_ctl" + elif [[ -e ${ROOT_MOUNT_DIR}/etc/init.d/qat_service ]]; then + rm "${ROOT_MOUNT_DIR}/etc/init.d/qat_service" + rm "${ROOT_MOUNT_DIR}/usr/local/bin/adf_ctl" + fi + rm -f "${ROOT_MOUNT_DIR}/etc/init.d/qat_service_vfs" + rm -f "${ROOT_MOUNT_DIR}/etc/modprobe.d/blacklist-qat-vfs.conf" + + rm -f "${ROOT_MOUNT_DIR}/usr/local/lib/libqat_s.so" + rm -f "${ROOT_MOUNT_DIR}/usr/local/lib/libusdm_drv_s.so" + rm -f "${ROOT_MOUNT_DIR}/etc/ld.so.conf.d/qat.conf" + ldconfig -r "${ROOT_MOUNT_DIR}" + + info "Removing config files" + rm -f "${ROOT_MOUNT_DIR}/etc/dh895xcc*.conf" + rm -f "${ROOT_MOUNT_DIR}/etc/c6xx*.conf" + rm -f "${ROOT_MOUNT_DIR}/etc/d15xx*.conf" + rm -f "${ROOT_MOUNT_DIR}/etc/c3xxx*.conf" + rm -f "${ROOT_MOUNT_DIR}/etc/200xx*.conf" + rm -f "${ROOT_MOUNT_DIR}/etc/udev/rules.d/00-qat.rules" + + mv -f "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/dh895xcc*.conf" "${ROOT_MOUNT_DIR}/etc/" 2>/dev/null || true + mv -f "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/c6xx*.conf" "${ROOT_MOUNT_DIR}/etc/" 2>/dev/null || true + mv -f "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/d15xx*.conf" "${ROOT_MOUNT_DIR}/etc/" 2>/dev/null || true + mv -f "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/c3xxx*.conf" "${ROOT_MOUNT_DIR}/etc/" 2>/dev/null || true + mv -f "${ROOT_MOUNT_DIR}/etc/qat_conf_backup/200xx*.conf" "${ROOT_MOUNT_DIR}/etc/" 2>/dev/null || true + + info "Removing drivers modules" + rm -rf "${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/updates/drivers/crypto/qat" + rm -f "${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/kernel/drivers/usdm_drv.ko" + rm -f "${ROOT_MOUNT_DIR}/lib/modules/$(uname -r)/kernel/drivers/qat_api.ko" + info "Creating module.dep file for QAT released kernel object" + depmod -a -b "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/depmod.d" + + if [[ $(lsmod | grep -c "usdm_drv|intel_qat") != "0" ]]; then + if [[ $(modinfo intel_qat | grep -c "updates") == "0" ]]; then + info "In-tree driver loaded" + info "Acceleration uninstall complete" + else + error "Some modules not removed properly" + error "Acceleration uninstall failed" + fi + else + info "Acceleration uninstall complete" + fi + if [[ ${numDh895xDevicesP} != 0 ]]; then + lsmod | grep qat_dh895xcc >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_dh895xcc >/dev/null 2>&1 || true + fi + if [[ ${numC62xDevicesP} != 0 ]]; then + lsmod | grep qat_c62x >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_c62x >/dev/null 2>&1 || true + fi + if [[ ${numD15xxDevicesP} != 0 ]]; then + lsmod | grep qat_d15xx >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_d15xx >/dev/null 2>&1 || true + fi + if [[ ${numC3xxxDevicesP} != 0 ]]; then + lsmod | grep qat_c3xxx >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_c3xxx >/dev/null 2>&1 || true + fi + if [[ ${num200xxDevicesP} != 0 ]]; then + lsmod | grep qat_200xx >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_200xx >/dev/null 2>&1 || true + fi + if [[ ${numDh895xDevicesV} != 0 ]]; then + lsmod | grep qat_dh895xccvf >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_dh895xccvf >/dev/null 2>&1 || true + fi + if [[ ${numC62xDevicesV} != 0 ]]; then + lsmod | grep qat_c62xvf >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_c62xvf >/dev/null 2>&1 || true + fi + if [[ ${numD15xxDevicesV} != 0 ]]; then + lsmod | grep qat_d15xxvf >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_d15xxvf >/dev/null 2>&1 || true + fi + if [[ ${numC3xxxDevicesV} != 0 ]]; then + lsmod | grep qat_c3xxxvf >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_c3xxxvf >/dev/null 2>&1 || true + fi + if [[ ${num200xxDevicesV} != 0 ]]; then + lsmod | grep qat_200xxvf >/dev/null 2>&1 || modprobe -d "${ROOT_MOUNT_DIR}" -C "${ROOT_MOUNT_DIR}/etc/modprobe.d" -b -q qat_200xxvf >/dev/null 2>&1 || true + fi + else + info "Acceleration package not installed" + fi +} + +_qat_sample_install() { + info "Installing samples" + if [[ -f ${QAT_INSTALL_DIR_CONTAINER}/quickassist/utilities/libusdm_drv/linux/build/linux_2.6/user_space/libusdm_drv.a ]]; then + ICP_ROOT="${QAT_INSTALL_DIR_CONTAINER}" make perf_user -C "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code" + cp "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code/performance/build/linux_2.6/user_space/cpa_sample_code" "${QAT_INSTALL_DIR_CONTAINER}/build" + ICP_ROOT="${QAT_INSTALL_DIR_CONTAINER}" KERNEL_SOURCE_ROOT="${KERNEL_SRC_DIR}" make perf_kernel -C "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code" + cp "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code/performance/build/linux_2.6/kernel_space/cpa_sample_code.ko" "${QAT_INSTALL_DIR_CONTAINER}/build" + else + error "No libusdm_drv library found - build the project (make all) before samples" + return "${RETCODE_ERROR}" + fi + + if [[ ! -d ${ROOT_MOUNT_DIR}/lib/firmware ]]; then + mkdir "${ROOT_MOUNT_DIR}/lib/firmware" + fi + + cp "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code/performance/compression/calgary" "${ROOT_MOUNT_DIR}/lib/firmware" + cp "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code/performance/compression/calgary32" "${ROOT_MOUNT_DIR}/lib/firmware" + cp "${QAT_INSTALL_DIR_CONTAINER}/quickassist/lookaside/access_layer/src/sample_code/performance/compression/canterbury" "${ROOT_MOUNT_DIR}/lib/firmware" + if [[ ! -z "${KERNEL_MOD_SIGN_CMD}" ]]; then + if [[ -f ${QAT_INSTALL_DIR_CONTAINER}/build/cpa_sample_code.ko ]]; then + echo "Need to sign sample code ${QAT_INSTALL_DIR_CONTAINER}/build/cpa_sample_code.ko." + "${KERNEL_MOD_SIGN_CMD}" "${QAT_INSTALL_DIR_CONTAINER}/build/cpa_sample_code.ko" + fi + fi + + install -D -m 750 "${QAT_INSTALL_DIR_CONTAINER}/build/cpa_sample_code" "${ROOT_MOUNT_DIR}/usr/local/bin/cpa_sample_code" + install -D -m 750 "${QAT_INSTALL_DIR_CONTAINER}/build/cpa_sample_code.ko" "${ROOT_MOUNT_DIR}/usr/local/bin/cpa_sample_code.ko" + info "cpa_sample_code installed under ${ROOT_MOUNT_DIR}/usr/local/bin directory" +} + +_qat_sample_uninstall() { + info "Uninstalling samples" + rm -f "${ROOT_MOUNT_DIR}/lib/firmware/calgary" + rm -f "${ROOT_MOUNT_DIR}/lib/firmware/calgary32" + rm -f "${ROOT_MOUNT_DIR}/lib/firmware/canterbury" + + rm -f "${ROOT_MOUNT_DIR}/usr/local/bin/cpa_sample_code" + rm -f "${ROOT_MOUNT_DIR}/usr/local/bin/cpa_sample_code.ko" +} diff --git a/kud/deployment_infra/installers/entrypoint-qat-driver-installer.sh b/kud/deployment_infra/installers/entrypoint-qat-driver-installer.sh new file mode 100755 index 00000000..f9221309 --- /dev/null +++ b/kud/deployment_infra/installers/entrypoint-qat-driver-installer.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +#set -x +source _common.sh +source _qat-driver-installer.sh + +# IMPORTANT: If the driver version is changed, review the QAT Makefile +# against _qat.sh. The steps in _qat.sh are from the Makefile and +# have been modified to run inside a container. +QAT_DRIVER_VERSION="${QAT_DRIVER_VERSION:-1.7.l.4.12.0-00011}" +QAT_DRIVER_DOWNLOAD_URL_DEFAULT="https://01.org/sites/default/files/downloads/qat${QAT_DRIVER_VERSION}.tar.gz" +QAT_DRIVER_DOWNLOAD_URL="${QAT_DRIVER_DOWNLOAD_URL:-$QAT_DRIVER_DOWNLOAD_URL_DEFAULT}" +QAT_DRIVER_ARCHIVE="$(basename "${QAT_DRIVER_DOWNLOAD_URL}")" +QAT_INSTALL_DIR_HOST="${QAT_INSTALL_DIR_HOST:-/opt/qat}" +QAT_INSTALL_DIR_CONTAINER="${QAT_INSTALL_DIR_CONTAINER:-/usr/local/qat}" +QAT_ENABLE_SRIOV="${QAT_ENABLE_SRIOV:-host}" +CACHE_FILE="${QAT_INSTALL_DIR_CONTAINER}/.cache" + +check_kernel_boot_parameter() { + if [[ $(grep -c intel_iommu=on /proc/cmdline) != "0" ]]; then + info "Found intel_iommu=on kernel boot parameter" + else + error "Missing intel_iommu=on kernel boot parameter" + exit "${RETCODE_ERROR}" + fi +} + +check_sriov_hardware_capabilities() { + if [[ $(lspci -vn -d 8086:0435 | grep -c SR-IOV) != "0" ]]; then + info "Found dh895xcc SR-IOV hardware capabilities" + elif [[ $(lspci -vn -d 8086:37c8 | grep -c SR-IOV) != "0" ]]; then + info "Found c6xx SR-IOV hardware capabilities" + elif [[ $(lspci -vn -d 8086:6f54 | grep -c SR-IOV) != "0" ]]; then + info "Found d15xx SR-IOV hardware capabilities" + elif [[ $(lspci -vn -d 8086:19e2 | grep -c SR-IOV) != "0" ]]; then + info "Found c3xxx SR-IOV hardware capabilities" + else + error "Missing SR-IOV hardware capabilities" + exit "${RETCODE_ERROR}" + fi +} + +download_qat_src() { + info "Downloading QAT source ... " + mkdir -p "${QAT_INSTALL_DIR_CONTAINER}" + pushd "${QAT_INSTALL_DIR_CONTAINER}" > /dev/null + curl -L -sS "${QAT_DRIVER_DOWNLOAD_URL}" -o "${QAT_DRIVER_ARCHIVE}" + tar xf "${QAT_DRIVER_ARCHIVE}" + popd > /dev/null +} + +build_qat_src() { + info "Building QAT source ... " + pushd "${QAT_INSTALL_DIR_CONTAINER}" > /dev/null + KERNEL_SOURCE_ROOT="${KERNEL_SRC_DIR}" ./configure --enable-icp-sriov="${QAT_ENABLE_SRIOV}" + make + popd > /dev/null +} + +install_qat() { + check_kernel_boot_parameter + check_sriov_hardware_capabilities + download_qat_src + build_qat_src + _qat_driver_install + _adf_ctl_install + _qat_service_install +} + +uninstall_qat() { + _adf_ctl_uninstall + _qat_service_shutdown + _qat_service_uninstall +} + +check_cached_version() { + info "Checking cached version" + if [[ ! -f "${CACHE_FILE}" ]]; then + info "Cache file ${CACHE_FILE} not found" + return "${RETCODE_ERROR}" + fi + # Source the cache file and check if the cached driver matches + # currently running kernel and driver versions. + . "${CACHE_FILE}" + if [[ "$(uname -r)" == "${CACHE_KERNEL_VERSION}" ]]; then + if [[ "${QAT_DRIVER_VERSION}" == "${CACHE_QAT_DRIVER_VERSION}" ]]; then + info "Found existing driver installation for kernel version $(uname -r) and driver version ${QAT_DRIVER_VERSION}" + return "${RETCODE_SUCCESS}" + fi + fi + return "${RETCODE_ERROR}" +} + +update_cached_version() { + cat >"${CACHE_FILE}"<<__EOF__ +CACHE_KERNEL_VERSION=$(uname -r) +CACHE_QAT_DRIVER_VERSION=${QAT_DRIVER_VERSION} +__EOF__ + + info "Updated cached version as:" + cat "${CACHE_FILE}" +} + +upgrade_driver() { + uninstall_qat + install_qat +} + +check_driver_started() { + _qat_check_started +} + +start_driver() { + _qat_service_start + _qat_check_started +} + +uninstall_driver() { + uninstall_qat + rm -f "${CACHE_FILE}" +} + +main() { + load_etc_os_release + local -r cmd="${1:-install}" + case $cmd in + install) + if ! check_cached_version; then + upgrade_driver + update_cached_version + fi + if ! check_driver_started; then + start_driver + fi + ;; + uninstall) + uninstall_driver + ;; + install-sample) + _qat_sample_install + ;; + uninstall-sample) + _qat_sample_uninstall + ;; + esac +} + +main "$@" diff --git a/kud/tests/qat.sh b/kud/tests/qat.sh index 8365f700..11fb6ca0 100755 --- a/kud/tests/qat.sh +++ b/kud/tests/qat.sh @@ -10,7 +10,7 @@ set -o pipefail -qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select(.status.capacity."qat.intel.com/cy2_dc2">="1") | .metadata.name') +qat_capable_nodes=$(kubectl get nodes -o json | jq -r '.items[] | select((.status.capacity."qat.intel.com/cy2_dc2"|tonumber)>=1) | .metadata.name') if [ -z "$qat_capable_nodes" ]; then echo "This test case cannot run. QAT device unavailable." QAT_ENABLED=False |