From 9837132fd9360f7d88d650b2ec6a318662e43d5c Mon Sep 17 00:00:00 2001 From: Akhila Kishore Date: Fri, 30 Aug 2019 16:39:00 -0700 Subject: Integrating NFD Daemonset with KuD Current NFD code in KuD is unused and obsolete. Integrating NFD as DaemonSet and updating test case for NFD. Added comments. Addressed comments and changed the matchExpression to kernel features. Changed operator from "In" to Gt, and values 4 to 3 better fit broader spectrum of O.S's. Adding exit conditon in case there's an error status. Signed-off-by: Akhila Kishore Issue-ID: MULTICLOUD-797 Change-Id: I454fb1998fc84e5f0d566f32b7dcfd85872c5183 --- kud/deployment_infra/images/nfd-master.yaml | 86 ++++++++++++++++++++++++ kud/deployment_infra/images/nfd-worker.yaml | 61 +++++++++++++++++ kud/deployment_infra/playbooks/configure-nfd.yml | 51 ++------------ kud/deployment_infra/playbooks/kud-vars.yml | 4 -- kud/hosting_providers/vagrant/installer.sh | 2 +- kud/tests/nfd.sh | 34 ++++++---- 6 files changed, 173 insertions(+), 65 deletions(-) create mode 100644 kud/deployment_infra/images/nfd-master.yaml create mode 100644 kud/deployment_infra/images/nfd-worker.yaml diff --git a/kud/deployment_infra/images/nfd-master.yaml b/kud/deployment_infra/images/nfd-master.yaml new file mode 100644 index 00000000..846bb753 --- /dev/null +++ b/kud/deployment_infra/images/nfd-master.yaml @@ -0,0 +1,86 @@ +# Refernce NFD Master DaemonSet - https://github.com/kubernetes-sigs/node-feature-discovery/blob/master/nfd-master.yaml.template +apiVersion: v1 +kind: Namespace +metadata: + name: node-feature-discovery # NFD namespace +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nfd-master + namespace: node-feature-discovery +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nfd-master +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nfd-master +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nfd-master +subjects: +- kind: ServiceAccount + name: nfd-master + namespace: node-feature-discovery +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nfd-master + name: nfd-master + namespace: node-feature-discovery +spec: + selector: + matchLabels: + app: nfd-master + template: + metadata: + labels: + app: nfd-master + spec: + serviceAccount: nfd-master + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + containers: + - env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/kubernetes_incubator/node-feature-discovery:v0.4.0 + name: nfd-master + command: + - "nfd-master" +--- +apiVersion: v1 +kind: Service +metadata: + name: nfd-master + namespace: node-feature-discovery +spec: + selector: + app: nfd-master + ports: + - protocol: TCP + port: 8080 + type: ClusterIP diff --git a/kud/deployment_infra/images/nfd-worker.yaml b/kud/deployment_infra/images/nfd-worker.yaml new file mode 100644 index 00000000..44bec5cd --- /dev/null +++ b/kud/deployment_infra/images/nfd-worker.yaml @@ -0,0 +1,61 @@ +# Reference to NFD worker DaemonSet - https://github.com/kubernetes-sigs/node-feature-discovery/blob/master/nfd-worker-daemonset.yaml.template +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: nfd-worker + name: nfd-worker + namespace: node-feature-discovery +spec: + selector: + matchLabels: + app: nfd-worker + template: + metadata: + labels: + app: nfd-worker + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: quay.io/kubernetes_incubator/node-feature-discovery:v0.4.0 + name: nfd-worker + command: + - "nfd-worker" + args: + - "--sleep-interval=60s" + - "--server=nfd-master:8080" + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + - name: source-d + mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: source-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/source.d/" + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" diff --git a/kud/deployment_infra/playbooks/configure-nfd.yml b/kud/deployment_infra/playbooks/configure-nfd.yml index 590212cf..32e7b38f 100644 --- a/kud/deployment_infra/playbooks/configure-nfd.yml +++ b/kud/deployment_infra/playbooks/configure-nfd.yml @@ -7,52 +7,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -- hosts: kube-node - tasks: - - name: Load kud variables - include_vars: - file: kud-vars.yml - - name: clone NFD repo - git: - repo: "{{ nfd_url }}" - dest: "{{ nfd_dest }}" - version: "{{ nfd_version }}" - force: yes - when: nfd_source_type == "source" - - name: build NFD image - become: yes - make: - chdir: "{{ nfd_dest }}" - - name: get NDF image name - become: yes - shell: "docker images | grep kubernetes_incubator | awk '{printf(\"%s:%s\\n\", $1,$2)}'" - register: nfd_image - - name: replace NFD image name - lineinfile: - path: "{{ nfd_dest }}/node-feature-discovery-{{ item }}.json.template" - regexp: "\"image\": \"quay.io/kubernetes_incubator.*i" - line: "\"image\": \"{{ nfd_image.stdout }}\"," - with_items: - - daemonset - - job - - name: copying rbac and daemonset files - fetch: - src: "{{ nfd_dest }}/{{ item }}" - dest: "/tmp/" - flat: yes - with_items: - - rbac.yaml - - node-feature-discovery-daemonset.json.template - - hosts: localhost - pre_tasks: - - name: Load kud variables - include_vars: - file: kud-vars.yml tasks: - - name: create service accounts - command: "/usr/local/bin/kubectl apply -f /tmp/{{ item }}" - with_items: - - rbac.yaml - - node-feature-discovery-daemonset.json.template + - name: Apply NFD Master Daemonset + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/nfd-master.yaml" + - name: Apply NFD Worker Daemonset + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/nfd-worker.yaml" diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 15e24ef1..a9910f8d 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -33,10 +33,6 @@ virtlet_url: "https://github.com/Mirantis/virtlet/releases/download/v{{ virtlet_ #virtlet_version: 68e11b8f1db2c78b063126899f0e60910700975d #virtlet_url: "https://github.com/Mirantis/virtlet" -nfd_dest: "{{ base_dest }}/nfd" -nfd_source_type: "source" -nfd_version: 175305b1ad73be7301ac94add475cec6fef797a9 -nfd_url: "https://github.com/kubernetes-incubator/node-feature-discovery" istio_dest: "{{ base_dest }}/istio" istio_source_type: "tarball" diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 51ca22e8..41b21f64 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -150,7 +150,7 @@ function install_addons { sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log if [[ "${testing_enabled}" == "true" ]]; then diff --git a/kud/tests/nfd.sh b/kud/tests/nfd.sh index b7e1f3e8..abf9f92b 100755 --- a/kud/tests/nfd.sh +++ b/kud/tests/nfd.sh @@ -15,40 +15,40 @@ set -o pipefail source _common_test.sh rm -f $HOME/*.yaml - pod_name=nfd-pod install_deps cat << POD > $HOME/$pod_name.yaml -apiVersion: - v1 +apiVersion: v1 kind: Pod metadata: name: $pod_name - labels: - env: test spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "feature.node.kubernetes.io/kernel-version.major" + operator: Gt + values: + - '3' containers: - - name: nginx - image: nginx -nodeSelector: - node.alpha.kubernetes-incubator.io/nfd-network-SRIOV: true + - name: with-node-affinity + image: gcr.io/google_containers/pause:2.0 POD - if $(kubectl version &>/dev/null); then labels=$(kubectl get nodes -o json | jq .items[].metadata.labels) echo $labels - if [[ $labels != *"node.alpha.kubernetes-incubator.io"* ]]; then + if [[ $labels != *"kubernetes.io"* ]]; then exit 1 fi - kubectl delete pod $pod_name --ignore-not-found=true --now while kubectl get pod $pod_name &>/dev/null; do sleep 5 done kubectl create -f $HOME/$pod_name.yaml --validate=false - for pod in $pod_name; do status_phase="" while [[ $status_phase != "Running" ]]; do @@ -57,9 +57,17 @@ if $(kubectl version &>/dev/null); then echo "$(date +%H:%M:%S) - $pod : $new_phase" status_phase=$new_phase fi + + if [[ $new_phase == "Running" ]]; then + echo " Test is complete.." + fi if [[ $new_phase == "Err"* ]]; then exit 1 fi done done + kubectl delete pod $pod_name + while kubectl get pod $pod_name &>/dev/null; do + sleep 5 + done fi -- cgit 1.2.3-korg