diff options
Diffstat (limited to 'kud/deployment_infra')
| -rw-r--r-- | kud/deployment_infra/galaxy-requirements.yml | 2 | ||||
| -rw-r--r-- | kud/deployment_infra/images/multus-daemonset.yml | 2 | ||||
| -rw-r--r-- | kud/deployment_infra/images/sriov-cni.yml | 45 | ||||
| -rw-r--r-- | kud/deployment_infra/images/sriov-daemonset.yml | 82 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/Debian.yml | 2 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml | 56 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-onap4k8s.yml | 55 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-sriov.yml | 29 | ||||
| -rwxr-xr-x | kud/deployment_infra/playbooks/install_iavf_drivers.sh | 105 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/kud-vars.yml | 8 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/preconfigure-sriov.yml | 118 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/sriov-nad.yml | 19 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/sriov_hardware_check.sh | 26 |
13 files changed, 544 insertions, 5 deletions
diff --git a/kud/deployment_infra/galaxy-requirements.yml b/kud/deployment_infra/galaxy-requirements.yml index 17ac1dc2..3191dc19 100644 --- a/kud/deployment_infra/galaxy-requirements.yml +++ b/kud/deployment_infra/galaxy-requirements.yml @@ -10,6 +10,6 @@ - src: andrewrothstein.go version: v2.1.15 - src: andrewrothstein.kubernetes-helm - version: v1.2.9 + version: v1.2.17 - src: geerlingguy.docker version: 2.5.2 diff --git a/kud/deployment_infra/images/multus-daemonset.yml b/kud/deployment_infra/images/multus-daemonset.yml index ff44a217..0c41a052 100644 --- a/kud/deployment_infra/images/multus-daemonset.yml +++ b/kud/deployment_infra/images/multus-daemonset.yml @@ -79,7 +79,7 @@ data: "delegates": [ { "cniVersion": "0.3.1", - "name": "default-cni-network", + "name": "cni0", "plugins": [ { "type": "flannel", diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml new file mode 100644 index 00000000..bd943d04 --- /dev/null +++ b/kud/deployment_infra/images/sriov-cni.yml @@ -0,0 +1,45 @@ +# SRIOV-CNI Release v1 +# Based on: +# https://github.com/intel/sriov-cni/blob/master/images/sriov-cni-daemonset.yaml +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-cni-ds-amd64 + namespace: kube-system + labels: + tier: node + app: sriov-cni +spec: + template: + metadata: + labels: + tier: node + app: sriov-cni + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: kube-sriov-cni + image: nfvpe/sriov-cni + securityContext: + privileged: true + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml new file mode 100644 index 00000000..72f33869 --- /dev/null +++ b/kud/deployment_infra/images/sriov-daemonset.yml @@ -0,0 +1,82 @@ +# SRIOV device CNI plugin +# Based on: +# https://github.com/intel/sriov-network-device-plugin/blob/master/images/sriovdp-daemonset.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [{ + "resourceName": "intel_sriov_700", + "selectors": { + "vendors": ["8086"], + "drivers": ["i40evf", "iavf"] + } + }] + } + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-device-plugin + namespace: kube-system + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-device-plugin-amd64 + namespace: kube-system + labels: + tier: node + app: sriovdp +spec: + template: + metadata: + labels: + tier: node + app: sriovdp + spec: + hostNetwork: true + hostPID: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: sriov-device-plugin + containers: + - name: kube-sriovdp + image: nfvpe/sriov-device-plugin + args: + - --log-dir=sriovdp + - --log-level=10 + securityContext: + privileged: true + volumeMounts: + - name: devicesock + mountPath: /var/lib/kubelet/ + readOnly: false + - name: log + mountPath: /var/log + - name: config-volume + mountPath: /etc/pcidp + volumes: + - name: devicesock + hostPath: + path: /var/lib/kubelet/ + - name: log + hostPath: + path: /var/log + - name: config-volume + configMap: + name: sriovdp-config + items: + - key: config.json + path: config.json diff --git a/kud/deployment_infra/playbooks/Debian.yml b/kud/deployment_infra/playbooks/Debian.yml index 96357fe2..b9725b2d 100644 --- a/kud/deployment_infra/playbooks/Debian.yml +++ b/kud/deployment_infra/playbooks/Debian.yml @@ -11,8 +11,6 @@ openvswitch_service: openvswitch-switch openvswitch_pkgs: - openvswitch-common - openvswitch-switch - - libopenvswitch - - openvswitch-datapath-dkms ovn_central_service: ovn-central ovn_central_pkgs: - ovn-central # <= 2.8.1-1 diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml new file mode 100644 index 00000000..6adaf2ee --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml @@ -0,0 +1,56 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-master + tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + + - name: Change the onap4k8s directory and run helm delete + command: /usr/local/bin/helm delete --purge multicloud-onap8ks + register: helm_delete + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: helm_delete.stdout_lines + + - name: Change the onap4k8s directory and delete the ona4k8s-ns namespace + command: /usr/local/bin/kubectl delete ns onap4k8s-ns + register: delete_onap_ns + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: delete_onap_ns.stdout_lines + + - name: Change the onap4k8s directory and make clean + command: /usr/bin/make clean + register: make_clean + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_clean.stdout_lines + + - name: Change the onap4k8s directory and make repo-stop + command: /usr/bin/make repo-stop + register: make_repo_stop + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_repo_stop.stdout_lines + + - name: clean multicloud-k8s path + file: + state: absent + path: /opt/multicloud diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s.yml b/kud/deployment_infra/playbooks/configure-onap4k8s.yml new file mode 100644 index 00000000..11729171 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-onap4k8s.yml @@ -0,0 +1,55 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-master + tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + + - name: Getting onap4k8s code in /opt folder + git: + repo: 'https://github.com/onap/multicloud-k8s.git' + dest: /opt/multicloud + + - name: install make package for ubuntu systems + apt: name=make state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install make package for centos systems + yum: name=make state=present update_cache=yes + when: ansible_distribution == "CentOS" + + - name: Change the onap4k8s directory and run the command make repo + command: /usr/bin/make repo + register: make_repo + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_repo.stdout_lines + + - name: Change the onap4k8s directory and run the command make all + command: /usr/bin/make all + register: make_all + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_all.stdout_lines + + - name: Change the onap4k8s directory and run the command helm install + command: /usr/local/bin/helm install dist/packages/multicloud-k8s-5.0.0.tgz --name multicloud-onap8ks --namespace onap4k8s-ns --set service.type=NodePort + register: helm_install + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: helm_install.stdout_lines diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml new file mode 100644 index 00000000..45f276c6 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-sriov.yml @@ -0,0 +1,29 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- import_playbook: preconfigure-sriov.yml + +- hosts: localhost + become: yes + tasks: + - debug: + var: SRIOV_NODE + - name: Apply Multus + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml" + when: SRIOV_NODE + - name: Apply SRIOV CNI + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml" + when: SRIOV_NODE + - name: Apply SRIOV DaemonSet + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml" + when: SRIOV_NODE + - name: Apply SRIOV Network Attachment definition + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml" + when: SRIOV_NODE diff --git a/kud/deployment_infra/playbooks/install_iavf_drivers.sh b/kud/deployment_infra/playbooks/install_iavf_drivers.sh new file mode 100755 index 00000000..7a54e9f2 --- /dev/null +++ b/kud/deployment_infra/playbooks/install_iavf_drivers.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Based on: +# https://gerrit.akraino.org/r/#/c/icn/+/1359/1/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh + +nic_models=(XL710 X722) +nic_drivers=(i40e) +device_checkers=(is_not_used is_driver_match is_model_match) + +function install_iavf_driver { + local ifname=$1 + + echo "Installing modules..." + echo "Installing i40evf blacklist file..." + mkdir -p "/etc/modprobe.d/" + echo "blacklist i40evf" > "/etc/modprobe.d/iavf-blacklist-i40evf.conf" + + kver=`uname -a | awk '{print $3}'` + install_mod_dir=/lib/modules/$kver/updates/drivers/net/ethernet/intel/iavf/ + echo "Installing driver in $install_mod_dir" + mkdir -p $install_mod_dir + cp iavf.ko $install_mod_dir + + echo "Installing kernel module i40evf..." + depmod -a + modprobe i40evf + modprobe iavf + + echo "Enabling VF on interface $ifname..." + echo "/sys/class/net/$ifname/device/sriov_numvfs" + echo '8' > /sys/class/net/$ifname/device/sriov_numvfs +} + +function is_not_used { + local ifname=$1 + route_info=`ip route show | grep $ifname` + if [ -z "$route_info" ]; then + return 1 + else + return 0 + fi +} + +function is_driver_match { + local ifname=$1 + driver=`cat /sys/class/net/$ifname/device/uevent | grep DRIVER | cut -f2 -d "="` + if [ ! -z "$driver" ]; then + for nic_driver in ${nic_drivers[@]}; do + if [ "$driver" = "$nic_driver" ]; then + return 1 + fi + done + fi + return 0 +} + +function is_model_match { + local ifname=$1 + pci_addr=`cat /sys/class/net/$ifname/device/uevent | grep PCI_SLOT_NAME | cut -f2 -d "=" | cut -f2,3 -d ":"` + if [ ! -z "$pci_addr" ]; then + for nic_model in ${nic_models[@]}; do + model_match=$(lspci | grep $pci_addr | grep $nic_model) + if [ ! -z "$model_match" ]; then + return 1 + fi + done + fi + return 0 +} + +function get_sriov_ifname { + for net_device in /sys/class/net/*/ ; do + if [ -e $net_device/device/sriov_numvfs ] ; then + ifname=$(basename $net_device) + for device_checker in ${device_checkers[@]}; do + eval $device_checker $ifname + if [ "$?" = "0" ]; then + ifname="" + break + fi + done + if [ ! -z "$ifname" ]; then + echo $ifname + return + fi + fi + done + echo '' +} + +if [ $# -ne 1 ] ; then + ifname=$(get_sriov_ifname) + if [ -z "$ifname" ]; then + echo "Cannot find Nic with SRIOV support." + else + install_iavf_driver $ifname + fi +else + ifname=$1 + if [ ! -e /sys/class/net/$ifname/device/sriov_numvfs ] ; then + echo "${ifname} is not a valid sriov interface" + else + install_iavf_driver $ifname + fi +fi diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index a9910f8d..2a25049a 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -39,8 +39,14 @@ istio_source_type: "tarball" istio_version: 1.0.3 istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz" +sriov_dest: "{{ base_dest }}/sriov" +driver_source_type: "tarball" +driver_version: 3.7.34 +driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ driver_version }}.tar.gz" +package: iavf-3.7.34 + go_version: '1.12.5' kubespray_version: 2.10.4 -helm_client_version: 2.9.1 +helm_client_version: 2.13.1 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634 ansible_version: 2.7.10 diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml new file mode 100644 index 00000000..fd16d935 --- /dev/null +++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml @@ -0,0 +1,118 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-node + become: yes + pre_tasks: + - name: Create SRIOV driver folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - sriov + - copy: + src: "{{ playbook_dir }}/sriov_hardware_check.sh" + dest: sriov + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x sriov_hardware_check.sh" + args: + chdir: "sriov" + warn: False + - name: Register SRIOV + shell: "echo {{ SRIOV | default(False) }}" + - name: Run the script and Re-evaluate the variable + command: sriov/sriov_hardware_check.sh + register: output + - set_fact: + _SRIOV: "{{ output.stdout }}" + - name: Recreate the conf file for every host + file: + path: /tmp/sriov.conf + state: absent + delegate_to: localhost + - lineinfile : > + dest=/tmp/sriov.conf + create=yes + line='{{_SRIOV}}' + delegate_to: localhost + - name: Clean the script and folder. + file: + path: sriov + state: absent + +# Run the following task only if the SRIOV is set to True +# i.e when SRIOV hardware is available +- hosts: localhost + become: yes + pre_tasks: + - name: Read SRIOV value from the conf file. + command: cat /tmp/sriov.conf + register: installer_output + become: yes + - set_fact: + SRIOV_NODE: "{{ installer_output.stdout }}" + - name: Load kud variables + include_vars: + file: kud-vars.yml + when: SRIOV_NODE + tasks: + - name: Create sriov folder + file: + state: directory + path: "{{ sriov_dest }}" + ignore_errors: yes + when: SRIOV_NODE + - name: Get SRIOV compatible driver + get_url: "url={{ driver_url }} dest=/tmp/{{ package }}.tar.gz" + when: SRIOV_NODE + - name: Extract sriov source code + unarchive: + src: "/tmp/{{ package }}.tar.gz" + dest: "{{ sriov_dest }}" + when: SRIOV_NODE + - name: Build the default target + make: + chdir: "/tmp/sriov/{{ package }}/src" + become: yes + when: SRIOV_NODE +# Copy all the driver and install script into target node +- hosts: kube-node + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + when: _SRIOV + tasks: + - name: create SRIOV driver folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - sriov_driver + when: _SRIOV + - copy: + src: "{{ sriov_dest }}/{{ package }}/src/iavf.ko" + dest: sriov_driver + remote_src: no + when: _SRIOV + - copy: + src: "{{ playbook_dir }}/install_iavf_drivers.sh" + dest: sriov_driver/install.sh + remote_src: no + when: _SRIOV + - name: Changing perm of "install.sh", adding "+x" + file: dest=sriov_driver/install.sh mode=a+x + when: _SRIOV + - name: Run a script with arguments + shell: ./install.sh + args: + chdir: "sriov_driver" + when: _SRIOV diff --git a/kud/deployment_infra/playbooks/sriov-nad.yml b/kud/deployment_infra/playbooks/sriov-nad.yml new file mode 100644 index 00000000..7670b700 --- /dev/null +++ b/kud/deployment_infra/playbooks/sriov-nad.yml @@ -0,0 +1,19 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-eno2 + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_700 +spec: + config: '{ + "type": "sriov", + "cniVersion": "0.3.1", + "ipam": { + "type": "host-local", + "subnet": "10.56.206.0/24", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "10.56.206.1" + } + }' diff --git a/kud/deployment_infra/playbooks/sriov_hardware_check.sh b/kud/deployment_infra/playbooks/sriov_hardware_check.sh new file mode 100644 index 00000000..662c28c8 --- /dev/null +++ b/kud/deployment_infra/playbooks/sriov_hardware_check.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail + +source /etc/environment + +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) +if [ -z "$ethernet_adpator_version" ]; then + echo "False" + exit 0 +fi +SRIOV_ENABLED=${ethernet_adpator_version:-"false"} +#checking for the right hardware version of NIC on the machine +if [ "$ethernet_adpator_version" == "XL710" ]; then + echo "True" +else + echo "False" +fi |