diff options
Diffstat (limited to 'kud')
| -rw-r--r-- | kud/deployment_infra/images/sriov-cni.yml | 9 | ||||
| -rw-r--r-- | kud/deployment_infra/images/sriov-daemonset.yml | 9 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-sriov.yml | 14 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/kud-vars.yml | 3 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/preconfigure-sriov.yml | 150 | ||||
| -rwxr-xr-x | kud/hosting_providers/containerized/installer.sh | 2 | ||||
| -rwxr-xr-x | kud/hosting_providers/vagrant/installer.sh | 2 |
7 files changed, 100 insertions, 89 deletions
diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml index 7503b872..570b00ee 100644 --- a/kud/deployment_infra/images/sriov-cni.yml +++ b/kud/deployment_infra/images/sriov-cni.yml @@ -21,6 +21,15 @@ spec: tier: node app: sriov-cni spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: feature.node.kubernetes.io/network-sriov.capable + operator: In + values: + - "true" hostNetwork: true nodeSelector: beta.kubernetes.io/arch: amd64 diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml index e392028d..41b1cbaa 100644 --- a/kud/deployment_infra/images/sriov-daemonset.yml +++ b/kud/deployment_infra/images/sriov-daemonset.yml @@ -46,6 +46,15 @@ spec: tier: node app: sriovdp spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: feature.node.kubernetes.io/network-sriov.capable + operator: In + values: + - "true" hostNetwork: true hostPID: true nodeSelector: diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml index 45f276c6..c0b7c9e0 100644 --- a/kud/deployment_infra/playbooks/configure-sriov.yml +++ b/kud/deployment_infra/playbooks/configure-sriov.yml @@ -9,21 +9,19 @@ ############################################################################## - import_playbook: preconfigure-sriov.yml - - hosts: localhost - become: yes + vars: + sriov_enabled: "{{ groups['kube-node'] | map('extract', hostvars, ['SRIOV_ENABLED']) | select() | list | length > 0 }}" tasks: - - debug: - var: SRIOV_NODE - name: Apply Multus shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml" - when: SRIOV_NODE + when: sriov_enabled - name: Apply SRIOV CNI shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml" - when: SRIOV_NODE + when: sriov_enabled - name: Apply SRIOV DaemonSet shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml" - when: SRIOV_NODE + when: sriov_enabled - name: Apply SRIOV Network Attachment definition shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml" - when: SRIOV_NODE + when: sriov_enabled diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 8011c901..51607020 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -63,7 +63,8 @@ helm_client_version: 3.2.4 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634 ansible_version: 2.9.7 -sriov_dest: "{{ base_dest }}/sriov" +sriov_pkgs: make,gcc +sriov_dest: "{{ base_dest }}/sriov_driver" sriov_driver_source_type: "tarball" sriov_driver_version: 3.7.34 sriov_driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ sriov_driver_version }}.tar.gz" diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml index 4c633ced..8c95aae8 100644 --- a/kud/deployment_infra/playbooks/preconfigure-sriov.yml +++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml @@ -8,111 +8,101 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -- hosts: kube-node - become: yes - pre_tasks: - - name: Create SRIOV driver folder in the target destination - file: - state: directory - path: "{{ item }}" - with_items: - - sriov - - copy: - src: "{{ playbook_dir }}/sriov_hardware_check.sh" - dest: sriov - - name: Changing perm of "sh", adding "+x" - shell: "chmod +x sriov_hardware_check.sh" - args: - chdir: "sriov" - warn: False - - name: Register SRIOV - shell: "echo {{ SRIOV | default(False) }}" - - name: Run the script and Re-evaluate the variable - command: sriov/sriov_hardware_check.sh - register: output - - set_fact: - _SRIOV: "{{ output.stdout }}" - - name: Recreate the conf file for every host - file: - path: /tmp/sriov.conf - state: absent - delegate_to: localhost - - lineinfile : > - dest=/tmp/sriov.conf - create=yes - line='{{_SRIOV}}' - delegate_to: localhost - - name: Clean the script and folder. - file: - path: sriov - state: absent - -# Run the following task only if the SRIOV is set to True -# i.e when SRIOV hardware is available - hosts: localhost become: yes pre_tasks: - - name: Read SRIOV value from the conf file. - command: cat /tmp/sriov.conf - register: installer_output - become: yes - - set_fact: - SRIOV_NODE: "{{ installer_output.stdout }}" - name: Load kud variables include_vars: file: kud-vars.yml - when: SRIOV_NODE tasks: - - name: Create sriov folder + - name: Create SRIOV dest folder file: state: directory path: "{{ sriov_dest }}" - ignore_errors: yes - when: SRIOV_NODE - - name: Get SRIOV compatible driver - get_url: "url={{ sriov_driver_url }} dest=/tmp/{{ sriov_package }}.tar.gz" - when: SRIOV_NODE - - name: Extract sriov source code - unarchive: - src: "/tmp/{{ sriov_package }}.tar.gz" - dest: "{{ sriov_dest }}" - when: SRIOV_NODE - - name: Build the default target - make: - chdir: "{{ sriov_dest }}/{{ sriov_package }}/src" - become: yes - when: SRIOV_NODE -# Copy all the driver and install script into target node + - name: Fetching SRIOV driver + block: + - name: Download SRIOV driver tarball + get_url: + url: "{{ sriov_driver_url }}" + dest: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz" + - hosts: kube-node become: yes pre_tasks: - name: Load kud variables include_vars: file: kud-vars.yml - when: _SRIOV tasks: - - name: create SRIOV driver folder in the target destination + - name: Create a destination for driver folder in the target's /tmp + file: + state: directory + path: "{{ item }}" + with_items: + - "{{ base_dest }}/sriov/{{ sriov_package }}" + - name: Create SRIOV dest folder + file: + state: directory + path: "sriov" + - name: Register SRIOV env variable + shell: "echo {{ SRIOV_ENABLED | default(False) }}" + - name: Copy SRIOV check script to target + copy: + src: "{{ playbook_dir }}/sriov_hardware_check.sh" + dest: sriov + mode: 0755 + - name: Run the script and re-evaluate the variable + command: "sriov/sriov_hardware_check.sh" + register: output + - debug: + var: output.stdout_lines + - set_fact: + SRIOV_ENABLED: "{{ output.stdout }}" + - debug: + var: output + - name: Clean the script and folder + file: + path: sriov + state: absent + - name: Install SRIOV compilation packges + package: + name: "{{ item }}" + state: present + with_items: "{{ sriov_pkgs }}" + when: SRIOV_ENABLED + - name: Extract SRIOV source code + unarchive: + src: "{{ sriov_dest }}/{{ sriov_package }}.tar.gz" + dest: "{{ base_dest }}/sriov" + when: SRIOV_ENABLED + - name: Build the SRIOV target + make: + chdir: "{{ base_dest }}/sriov/{{ sriov_package }}/src" + when: SRIOV_ENABLED + - name: Create SRIOV driver folder in the target destination file: state: directory path: "{{ item }}" with_items: - sriov_driver - when: _SRIOV - - copy: - src: "{{ sriov_dest }}/{{ sriov_package }}/src/iavf.ko" + when: SRIOV_ENABLED + - name: Copy SRIOV module to target destination + copy: + src: "{{ base_dest }}/sriov/{{ sriov_package }}/src/iavf.ko" dest: sriov_driver - remote_src: no - when: _SRIOV - - copy: + remote_src: yes + when: SRIOV_ENABLED + - name: Copy SRIOV install script to target + copy: src: "{{ playbook_dir }}/install_iavf_drivers.sh" dest: sriov_driver/install.sh - remote_src: no - when: _SRIOV - - name: Changing perm of "install.sh", adding "+x" - file: dest=sriov_driver/install.sh mode=a+x - when: _SRIOV - - name: Run a script with arguments + mode: 0755 + when: SRIOV_ENABLED + - name: Run the install script with arguments shell: ./install.sh args: chdir: "sriov_driver" - when: _SRIOV + when: SRIOV_ENABLED + - name: Clean the SRIOV folder + file: + path: "{{ base_dest }}/sriov" + state: absent diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 6fd7c915..226f4568 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -120,6 +120,8 @@ function install_addons { ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-kud.yml | \ tee $cluster_log/setup-kud.log + # The order of KUD_ADDONS is important: some plugins (sriov, qat) + # require nfd to be enabled. for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 94f3c58c..53164989 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -157,6 +157,8 @@ function install_addons { _install_ansible sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log + # The order of KUD_ADDONS is important: some plugins (sriov, qat) + # require nfd to be enabled. for addon in ${KUD_ADDONS:-topology-manager virtlet ovn4nfv nfd sriov qat optane cmk}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log |