summaryrefslogtreecommitdiffstats
path: root/kud
diff options
context:
space:
mode:
authorTodd Malsbary <todd.malsbary@intel.com>2020-10-01 14:17:59 -0700
committerTodd Malsbary <todd.malsbary@intel.com>2020-10-05 13:13:06 -0700
commit61dbe22ebb42279e388f53c557c267d43a774ed2 (patch)
treee32aae2483599a1d7dda614217656c96c19db9cc /kud
parente70a776fd2d16d7d386af0354afd3e086068ef00 (diff)
Build and deploy sriov module only on supported hosts.
Building on the target host fixes a couple issues: - In the containerized installer, the container image does not include the necessary kernel headers to build the module. - The build and target host must have the same kernel version. There is no guarantee of this. The deploy uses NFD, similar to the QAT playbook. Issue-ID: MULTICLOUD-1228 Signed-off-by: Todd Malsbary <todd.malsbary@intel.com> Change-Id: I58705b73b8ce6d381b4649d5a20b8644e51e1b13
Diffstat (limited to 'kud')
-rw-r--r--kud/deployment_infra/images/sriov-cni.yml9
-rw-r--r--kud/deployment_infra/images/sriov-daemonset.yml9
-rw-r--r--kud/deployment_infra/playbooks/configure-sriov.yml14
-rw-r--r--kud/deployment_infra/playbooks/kud-vars.yml3
-rw-r--r--kud/deployment_infra/playbooks/preconfigure-sriov.yml150
-rwxr-xr-xkud/hosting_providers/containerized/installer.sh2
-rwxr-xr-xkud/hosting_providers/vagrant/installer.sh2
7 files changed, 100 insertions, 89 deletions
diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml
index 7503b872..570b00ee 100644
--- a/kud/deployment_infra/images/sriov-cni.yml
+++ b/kud/deployment_infra/images/sriov-cni.yml
@@ -21,6 +21,15 @@ spec:
tier: node
app: sriov-cni
spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: feature.node.kubernetes.io/network-sriov.capable
+ operator: In
+ values:
+ - "true"
hostNetwork: true
nodeSelector:
beta.kubernetes.io/arch: amd64
diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml
index e392028d..41b1cbaa 100644
--- a/kud/deployment_infra/images/sriov-daemonset.yml
+++ b/kud/deployment_infra/images/sriov-daemonset.yml
@@ -46,6 +46,15 @@ spec:
tier: node
app: sriovdp
spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: feature.node.kubernetes.io/network-sriov.capable
+ operator: In
+ values:
+ - "true"
hostNetwork: true
hostPID: true
nodeSelector:
diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml
index 45f276c6..c0b7c9e0 100644
--- a/kud/deployment_infra/playbooks/configure-sriov.yml
+++ b/kud/deployment_infra/playbooks/configure-sriov.yml
@@ -9,21 +9,19 @@
##############################################################################
- import_playbook: preconfigure-sriov.yml
-
- hosts: localhost
- become: yes
+ vars:
+ sriov_enabled: "{{ groups['kube-node'] | map('extract', hostvars, ['SRIOV_ENABLED']) | select() | list | length > 0 }}"
tasks:
- - debug:
- var: SRIOV_NODE
- name: Apply Multus
shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml"
- when: SRIOV_NODE
+ when: sriov_enabled
- name: Apply SRIOV CNI
shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml"
- when: SRIOV_NODE
+ when: sriov_enabled
- name: Apply SRIOV DaemonSet
shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml"
- when: SRIOV_NODE
+ when: sriov_enabled
- name: Apply SRIOV Network Attachment definition
shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml"
- when: SRIOV_NODE
+ when: sriov_enabled
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 30e54f03..8f8ff5c3 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -62,7 +62,8 @@ helm_client_version: 2.13.1
# kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634
ansible_version: 2.9.7
-sriov_dest: "{{ base_dest }}/sriov"
+sriov_pkgs: make,gcc
+sriov_dest: "{{ base_dest }}/sriov_driver"
sriov_driver_source_type: "tarball"
sriov_driver_version: 3.7.34
sriov_driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ sriov_driver_version }}.tar.gz"
diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml
index 4c633ced..8c95aae8 100644
--- a/kud/deployment_infra/playbooks/preconfigure-sriov.yml
+++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml
@@ -8,111 +8,101 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-- hosts: kube-node
- become: yes
- pre_tasks:
- - name: Create SRIOV driver folder in the target destination
- file:
- state: directory
- path: "{{ item }}"
- with_items:
- - sriov
- - copy:
- src: "{{ playbook_dir }}/sriov_hardware_check.sh"
- dest: sriov
- - name: Changing perm of "sh", adding "+x"
- shell: "chmod +x sriov_hardware_check.sh"
- args:
- chdir: "sriov"
- warn: False
- - name: Register SRIOV
- shell: "echo {{ SRIOV | default(False) }}"
- - name: Run the script and Re-evaluate the variable
- command: sriov/sriov_hardware_check.sh
- register: output
- - set_fact:
- _SRIOV: "{{ output.stdout }}"
- - name: Recreate the conf file for every host
- file:
- path: /tmp/sriov.conf
- state: absent
- delegate_to: localhost
- - lineinfile : >
- dest=/tmp/sriov.conf
- create=yes
- line='{{_SRIOV}}'
- delegate_to: localhost
- - name: Clean the script and folder.
- file:
- path: sriov
- state: absent
-
-# Run the following task only if the SRIOV is set to True
-# i.e when SRIOV hardware is available
- hosts: localhost
become: yes
pre_tasks:
- - name: Read SRIOV value from the conf file.
- command: cat /tmp/sriov.conf
- register: installer_output
- become: yes
- - set_fact:
- SRIOV_NODE: "{{ installer_output.stdout }}"
- name: Load kud variables
include_vars:
file: kud-vars.yml
- when: SRIOV_NODE
tasks:
- - name: Create sriov folder
+ - name: Create SRIOV dest folder
file:
state: directory
path: "{{ sriov_dest }}"
- ignore_errors: yes
- when: SRIOV_NODE
- - name: Get SRIOV compatible driver
- get_url: "url={{ sriov_driver_url }} dest=/tmp/{{ sriov_package }}.tar.gz"
- when: SRIOV_NODE
- - name: Extract sriov source code
- unarchive:
- src: "/tmp/{{ sriov_package }}.tar.gz"
- dest: "{{ sriov_dest }}"
- when: SRIOV_NODE
- - name: Build the default target
- make:
- chdir: "{{ sriov_dest }}/{{ sriov_package }}/src"
- become: yes
- when: SRIOV_NODE
-# Copy all the driver and install script into target node
+ - name: Fetching SRIOV driver
+ block:
+ - name: Download SRIOV driver tarball
+ get_url:
+ url: "{{ sriov_driver_url }}"
+ dest: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz"
+
- hosts: kube-node
become: yes
pre_tasks:
- name: Load kud variables
include_vars:
file: kud-vars.yml
- when: _SRIOV
tasks:
- - name: create SRIOV driver folder in the target destination
+ - name: Create a destination for driver folder in the target's /tmp
+ file:
+ state: directory
+ path: "{{ item }}"
+ with_items:
+ - "{{ base_dest }}/sriov/{{ sriov_package }}"
+ - name: Create SRIOV dest folder
+ file:
+ state: directory
+ path: "sriov"
+ - name: Register SRIOV env variable
+ shell: "echo {{ SRIOV_ENABLED | default(False) }}"
+ - name: Copy SRIOV check script to target
+ copy:
+ src: "{{ playbook_dir }}/sriov_hardware_check.sh"
+ dest: sriov
+ mode: 0755
+ - name: Run the script and re-evaluate the variable
+ command: "sriov/sriov_hardware_check.sh"
+ register: output
+ - debug:
+ var: output.stdout_lines
+ - set_fact:
+ SRIOV_ENABLED: "{{ output.stdout }}"
+ - debug:
+ var: output
+ - name: Clean the script and folder
+ file:
+ path: sriov
+ state: absent
+ - name: Install SRIOV compilation packges
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items: "{{ sriov_pkgs }}"
+ when: SRIOV_ENABLED
+ - name: Extract SRIOV source code
+ unarchive:
+ src: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz"
+ dest: "{{ base_dest }}/sriov"
+ when: SRIOV_ENABLED
+ - name: Build the SRIOV target
+ make:
+ chdir: "{{ base_dest }}/sriov/{{ sriov_package }}/src"
+ when: SRIOV_ENABLED
+ - name: Create SRIOV driver folder in the target destination
file:
state: directory
path: "{{ item }}"
with_items:
- sriov_driver
- when: _SRIOV
- - copy:
- src: "{{ sriov_dest }}/{{ sriov_package }}/src/iavf.ko"
+ when: SRIOV_ENABLED
+ - name: Copy SRIOV module to target destination
+ copy:
+ src: "{{ base_dest }}/sriov/{{ sriov_package }}/src/iavf.ko"
dest: sriov_driver
- remote_src: no
- when: _SRIOV
- - copy:
+ remote_src: yes
+ when: SRIOV_ENABLED
+ - name: Copy SRIOV install script to target
+ copy:
src: "{{ playbook_dir }}/install_iavf_drivers.sh"
dest: sriov_driver/install.sh
- remote_src: no
- when: _SRIOV
- - name: Changing perm of "install.sh", adding "+x"
- file: dest=sriov_driver/install.sh mode=a+x
- when: _SRIOV
- - name: Run a script with arguments
+ mode: 0755
+ when: SRIOV_ENABLED
+ - name: Run the install script with arguments
shell: ./install.sh
args:
chdir: "sriov_driver"
- when: _SRIOV
+ when: SRIOV_ENABLED
+ - name: Clean the SRIOV folder
+ file:
+ path: "{{ base_dest }}/sriov"
+ state: absent
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index ae16b1dd..c18829a5 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -119,6 +119,8 @@ function install_addons {
ansible-playbook $verbose -i \
$kud_inventory $kud_playbooks/configure-kud.yml | \
tee $cluster_log/setup-kud.log
+ # The order of KUD_ADDONS is important: some plugins (sriov, qat)
+ # require nfd to be enabled.
for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
echo "Deploying $addon using configure-$addon.yml playbook.."
ansible-playbook $verbose -i \
diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh
index 71e4d8b7..0ebc3313 100755
--- a/kud/hosting_providers/vagrant/installer.sh
+++ b/kud/hosting_providers/vagrant/installer.sh
@@ -155,6 +155,8 @@ function install_addons {
_install_ansible
sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors
ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log
+ # The order of KUD_ADDONS is important: some plugins (sriov, qat)
+ # require nfd to be enabled.
for addon in ${KUD_ADDONS:-topology-manager virtlet ovn4nfv nfd sriov qat optane cmk}; do
echo "Deploying $addon using configure-$addon.yml playbook.."
ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log