Build and deploy sriov module only on supported hosts.

Building on the target host fixes a couple issues:
- In the containerized installer, the container image does not include
  the necessary kernel headers to build the module.
- The build and target host must have the same kernel version.  There
  is no guarantee of this.

The deploy uses NFD, similar to the QAT playbook.

Issue-ID: MULTICLOUD-1228
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I58705b73b8ce6d381b4649d5a20b8644e51e1b13

7 files changed, 100 insertions, 89 deletions

diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml
index 7503b872..570b00ee 100644
--- a/kud/deployment_infra/images/sriov-cni.yml
+++ b/kud/deployment_infra/images/sriov-cni.yml
@@ -21,6 +21,15 @@ spec:
         tier: node
         app: sriov-cni
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: feature.node.kubernetes.io/network-sriov.capable
+                operator: In
+                values:
+                - "true"
       hostNetwork: true
       nodeSelector:
         beta.kubernetes.io/arch: amd64
diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml
index e392028d..41b1cbaa 100644
--- a/kud/deployment_infra/images/sriov-daemonset.yml
+++ b/kud/deployment_infra/images/sriov-daemonset.yml
@@ -46,6 +46,15 @@ spec:
         tier: node
         app: sriovdp
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: feature.node.kubernetes.io/network-sriov.capable
+                operator: In
+                values:
+                - "true"
       hostNetwork: true
       hostPID: true
       nodeSelector:
diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml
index 45f276c6..c0b7c9e0 100644
--- a/kud/deployment_infra/playbooks/configure-sriov.yml
+++ b/kud/deployment_infra/playbooks/configure-sriov.yml
@@ -9,21 +9,19 @@
 ##############################################################################
 
 - import_playbook: preconfigure-sriov.yml
-
 - hosts: localhost
-  become: yes
+  vars:
+    sriov_enabled: "{{ groups['kube-node'] | map('extract', hostvars, ['SRIOV_ENABLED']) | select() | list | length > 0 }}"
   tasks:
-    - debug:
-        var: SRIOV_NODE
     - name: Apply Multus
       shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml"
-      when: SRIOV_NODE
+      when: sriov_enabled
     - name: Apply SRIOV CNI
       shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml"
-      when: SRIOV_NODE
+      when: sriov_enabled
     - name: Apply SRIOV DaemonSet
       shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml"
-      when: SRIOV_NODE
+      when: sriov_enabled
     - name: Apply SRIOV Network Attachment definition
       shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml"
-      when: SRIOV_NODE
+      when: sriov_enabled
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 30e54f03..8f8ff5c3 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -62,7 +62,8 @@ helm_client_version: 2.13.1
 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634
 ansible_version: 2.9.7
 
-sriov_dest: "{{ base_dest }}/sriov"
+sriov_pkgs: make,gcc
+sriov_dest: "{{ base_dest }}/sriov_driver"
 sriov_driver_source_type: "tarball"
 sriov_driver_version: 3.7.34
 sriov_driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ sriov_driver_version }}.tar.gz"
diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml
index 4c633ced..8c95aae8 100644
--- a/kud/deployment_infra/playbooks/preconfigure-sriov.yml
+++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml
@@ -8,111 +8,101 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-- hosts: kube-node
-  become: yes
-  pre_tasks:
-    - name: Create SRIOV driver folder in the target destination
-      file:
-        state: directory
-        path: "{{ item }}"
-      with_items:
-        - sriov
-    - copy:
-        src: "{{ playbook_dir }}/sriov_hardware_check.sh"
-        dest: sriov
-    - name: Changing perm of "sh", adding "+x"
-      shell: "chmod +x sriov_hardware_check.sh"
-      args:
-        chdir: "sriov"
-        warn: False
-    - name: Register SRIOV
-      shell: "echo {{ SRIOV | default(False) }}"
-    - name: Run the script and Re-evaluate the variable
-      command: sriov/sriov_hardware_check.sh
-      register: output
-    - set_fact:
-        _SRIOV: "{{ output.stdout }}"
-    - name: Recreate the conf file for every host
-      file:
-        path: /tmp/sriov.conf
-        state: absent
-      delegate_to: localhost
-    - lineinfile : >
-       dest=/tmp/sriov.conf
-       create=yes
-       line='{{_SRIOV}}'
-      delegate_to: localhost
-    - name: Clean the script and folder.
-      file:
-        path: sriov
-        state: absent
-
-# Run the following task only if the SRIOV is set to True
-# i.e when SRIOV hardware is available
 - hosts: localhost
   become: yes
   pre_tasks:
-    - name: Read SRIOV value from the conf file.
-      command: cat /tmp/sriov.conf
-      register: installer_output
-      become: yes
-    - set_fact:
-        SRIOV_NODE: "{{ installer_output.stdout }}"
     - name: Load kud variables
       include_vars:
         file: kud-vars.yml
-      when: SRIOV_NODE
   tasks:
-    - name: Create sriov folder
+    - name: Create SRIOV dest folder
       file:
         state: directory
         path: "{{ sriov_dest }}"
-      ignore_errors: yes
-      when: SRIOV_NODE
-    - name: Get SRIOV compatible driver
-      get_url: "url={{ sriov_driver_url }}  dest=/tmp/{{ sriov_package }}.tar.gz"
-      when: SRIOV_NODE
-    - name: Extract sriov source code
-      unarchive:
-         src: "/tmp/{{ sriov_package }}.tar.gz"
-         dest: "{{ sriov_dest }}"
-      when: SRIOV_NODE
-    - name: Build the default target
-      make:
-        chdir: "{{ sriov_dest }}/{{ sriov_package }}/src"
-      become: yes
-      when: SRIOV_NODE
-# Copy all the driver and install script into target node
+    - name: Fetching SRIOV driver
+      block:
+      - name: Download SRIOV driver tarball
+        get_url:
+          url: "{{ sriov_driver_url }}"
+          dest: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz"
+
 - hosts: kube-node
   become: yes
   pre_tasks:
     - name: Load kud variables
       include_vars:
         file: kud-vars.yml
-      when: _SRIOV
   tasks:
-    - name: create SRIOV driver folder in the target destination
+    - name: Create a destination for driver folder in the target's /tmp
+      file:
+        state: directory
+        path: "{{ item }}"
+      with_items:
+        - "{{ base_dest }}/sriov/{{ sriov_package }}"
+    - name: Create SRIOV dest folder
+      file:
+        state: directory
+        path: "sriov"
+    - name: Register SRIOV env variable
+      shell: "echo {{ SRIOV_ENABLED | default(False) }}"
+    - name: Copy SRIOV check script to target
+      copy:
+        src: "{{ playbook_dir }}/sriov_hardware_check.sh"
+        dest: sriov
+        mode: 0755
+    - name: Run the script and re-evaluate the variable
+      command: "sriov/sriov_hardware_check.sh"
+      register: output
+    - debug:
+        var: output.stdout_lines
+    - set_fact:
+        SRIOV_ENABLED: "{{ output.stdout }}"
+    - debug:
+        var: output
+    - name: Clean the script and folder
+      file:
+        path: sriov
+        state: absent
+    - name: Install SRIOV compilation packges
+      package:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ sriov_pkgs }}"
+      when: SRIOV_ENABLED
+    - name: Extract SRIOV source code
+      unarchive:
+         src: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz"
+         dest: "{{ base_dest }}/sriov"
+      when: SRIOV_ENABLED
+    - name: Build the SRIOV target
+      make:
+        chdir: "{{ base_dest }}/sriov/{{ sriov_package }}/src"
+      when: SRIOV_ENABLED
+    - name: Create SRIOV driver folder in the target destination
       file:
         state: directory
         path: "{{ item }}"
       with_items:
         - sriov_driver
-      when: _SRIOV
-    - copy:
-        src: "{{ sriov_dest }}/{{ sriov_package }}/src/iavf.ko"
+      when: SRIOV_ENABLED
+    - name: Copy SRIOV module to target destination
+      copy:
+        src: "{{ base_dest }}/sriov/{{ sriov_package }}/src/iavf.ko"
         dest: sriov_driver
-        remote_src: no
-      when: _SRIOV
-    - copy:
+        remote_src: yes
+      when: SRIOV_ENABLED
+    - name: Copy SRIOV install script to target
+      copy:
         src: "{{ playbook_dir }}/install_iavf_drivers.sh"
         dest: sriov_driver/install.sh
-        remote_src: no
-      when: _SRIOV
-    - name: Changing perm of "install.sh", adding "+x"
-      file: dest=sriov_driver/install.sh mode=a+x
-      when: _SRIOV
-    - name: Run a script with arguments
+        mode: 0755
+      when: SRIOV_ENABLED
+    - name: Run the install script with arguments
       shell: ./install.sh
       args:
         chdir: "sriov_driver"
-      when: _SRIOV
+      when: SRIOV_ENABLED
+    - name: Clean the SRIOV folder
+      file:
+        path: "{{ base_dest }}/sriov"
+        state: absent
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index ae16b1dd..c18829a5 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -119,6 +119,8 @@ function install_addons {
     ansible-playbook $verbose -i \
         $kud_inventory $kud_playbooks/configure-kud.yml | \
         tee $cluster_log/setup-kud.log
+    # The order of KUD_ADDONS is important: some plugins (sriov, qat)
+    # require nfd to be enabled.
     for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do
         echo "Deploying $addon using configure-$addon.yml playbook.."
         ansible-playbook $verbose -i \
diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh
index 71e4d8b7..0ebc3313 100755
--- a/kud/hosting_providers/vagrant/installer.sh
+++ b/kud/hosting_providers/vagrant/installer.sh
@@ -155,6 +155,8 @@ function install_addons {
     _install_ansible
     sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors
     ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log
+    # The order of KUD_ADDONS is important: some plugins (sriov, qat)
+    # require nfd to be enabled.
     for addon in ${KUD_ADDONS:-topology-manager virtlet ovn4nfv nfd sriov qat optane cmk}; do
         echo "Deploying $addon using configure-$addon.yml playbook.."
         ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log