aboutsummaryrefslogtreecommitdiffstats
path: root/kud/deployment_infra
diff options
context:
space:
mode:
authorLiang Ding <liang.ding@intel.com>2019-10-30 23:47:35 -0700
committerLiang Ding <liang.ding@intel.com>2020-05-05 15:41:48 +0000
commitcc10b9aff3fd16df62c0655ec1626624ad2e2fc0 (patch)
tree506a942430b6eca7058b5549e445125b98f1e5ab /kud/deployment_infra
parent501d62d194a42c5c85f22d8208a4e1dd6d985492 (diff)
add cmk in KuD
- deploy cmk related pods - untaint compute nodes if necessary - run cmk unit tests: allocate CPUs from exclusive and shared pools - deploy a testing nginx pod along with cmk testing pods - preset 1/2 CPUs for shared/exlusive pools to fit CI server machines users can adjust the parameters to meet their own requirements Test Results: - many rounds of vagrant/5 VMs(controller01/02/03 and compute01/02) based test are all OK - 14 rounds tests on my local server (S2600WFQ (36C/72T) )and PC(HP Z228 (4C/4T)) with all-in-one bare metal deployment are all OK - CI(a 4C/4T machine) results of latest patch set also show that the test of bare metal deployment is OK - NOTE: both my local test and CI use the same testing method of calling aio.sh after applying the latest patch set. Change-Id: I046a4a63b94f92f23347ab76c21a661521e01119 Issue-ID: MULTICLOUD-879 Signed-off-by: Liang Ding <liang.ding@intel.com>
Diffstat (limited to 'kud/deployment_infra')
-rw-r--r--kud/deployment_infra/images/cmk.yaml294
-rw-r--r--kud/deployment_infra/playbooks/configure-cmk.yml107
-rw-r--r--kud/deployment_infra/playbooks/kud-vars.yml16
-rw-r--r--kud/deployment_infra/playbooks/preconfigure-cmk.yml62
4 files changed, 479 insertions, 0 deletions
diff --git a/kud/deployment_infra/images/cmk.yaml b/kud/deployment_infra/images/cmk.yaml
new file mode 100644
index 00000000..4c048a42
--- /dev/null
+++ b/kud/deployment_infra/images/cmk.yaml
@@ -0,0 +1,294 @@
+---
+# Source: cmk/templates/serviceaccount.yml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: cmk-custom-resource-definition-controller
+ namespace: kube-system
+rules:
+- apiGroups: ["intel.com"]
+ resources: ["*"]
+ verbs: ["*"]
+- apiGroups: ["apiextensions.k8s.io"]
+ resources: ["customresourcedefinitions", "customresourcedefinitions.extensions"]
+ verbs: ["*"]
+---
+# Source: cmk/templates/rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: cmk-daemonset-controller
+ namespace: kube-system
+rules:
+- apiGroups: ["extensions"]
+ resources: ["daemonsets", "daemonsets.extensions"]
+ verbs: ["*"]
+---
+# Source: cmk/templates/rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: cmk-third-party-resource-controller
+ namespace: kube-system
+rules:
+- apiGroups: ["cmk.intel.com"]
+ resources: ["*"]
+ verbs: ["*"]
+- apiGroups: ["extensions"]
+ resources: ["thirdpartyresources", "thirdpartyresources.extensions"]
+ verbs: ["*"]
+---
+# Source: cmk/templates/rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: cmk-version-controller
+ namespace: kube-system
+rules:
+ - nonResourceURLs: ["*"]
+ verbs:
+ - get
+---
+# Source: cmk/templates/rbac.yml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: cmk-role-binding-version
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cmk-version-controller
+subjects:
+- kind: ServiceAccount
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/rbac.yml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: cmk-role-binding-daemonset
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cmk-daemonset-controller
+subjects:
+- kind: ServiceAccount
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/rbac.yml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: cmk-role-binding-node
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: system:node
+subjects:
+- kind: ServiceAccount
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/rbac.yml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: cmk-role-binding-tpr
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cmk-third-party-resource-controller
+subjects:
+- kind: ServiceAccount
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/rbac.yml
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: cmk-role-binding-crd
+ namespace: kube-system
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cmk-custom-resource-definition-controller
+subjects:
+- kind: ServiceAccount
+ name: cmk
+ namespace: kube-system
+---
+# Source: cmk/templates/daemonset.yml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: cmk
+ labels:
+ app: cmk
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ name: cmk
+ template:
+ metadata:
+ labels:
+ name: cmk
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ serviceAccountName: cmk
+ tolerations:
+ - key: cmk
+ operator: Exists
+ containers:
+ - name: reconcile
+ image: localhost:5000/cmk:v1.4.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CMK_RECONCILE_SLEEP_TIME
+ value: '60'
+ - name: CMK_PROC_FS
+ value: /proc
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ command: ["/bin/bash", "-c"]
+ args:
+ - "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- reconcile --interval=$CMK_RECONCILE_SLEEP_TIME --publish"
+ volumeMounts:
+ - mountPath: /proc
+ name: host-proc
+ readOnly: false
+ - mountPath: /etc/cmk
+ name: cmk-conf-dir
+ - mountPath: /opt/bin
+ name: cmk-install-dir
+ - name: nodereport
+ image: localhost:5000/cmk:v1.4.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CMK_NODE_REPORT_SLEEP_TIME
+ value: '60'
+ - name: CMK_PROC_FS
+ value: /proc
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ command: ["/bin/bash", "-c"]
+ args:
+ - "/cmk/cmk.py isolate --pool=infra /cmk/cmk.py -- node-report --interval=$CMK_NODE_REPORT_SLEEP_TIME --publish"
+ volumeMounts:
+ - mountPath: /proc
+ name: host-proc
+ readOnly: false
+ - mountPath: /etc/cmk
+ name: cmk-conf-dir
+ - mountPath: /opt/bin
+ name: cmk-install-dir
+ initContainers:
+ - name: init
+ image: localhost:5000/cmk:v1.4.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CMK_PROC_FS
+ value: "/proc"
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ command: ["/bin/bash", "-c"]
+ args:
+ - "/cmk/cmk.py init --conf-dir=/etc/cmk --exclusive-mode=packed --num-exclusive-cores=0 --shared-mode=packed --num-shared-cores=0"
+ volumeMounts:
+ - mountPath: /proc
+ name: host-proc
+ readOnly: false
+ - mountPath: /etc/cmk
+ name: cmk-conf-dir
+ readOnly: false
+ - mountPath: /opt/bin
+ name: cmk-install-dir
+ readOnly: false
+ - name: discover
+ image: localhost:5000/cmk:v1.4.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CMK_PROC_FS
+ value: /proc
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ command: ["/bin/bash", "-c"]
+ args:
+# - "echo -en '\n'; ls -a /etc/cmk; sleep 10;"
+ - "until [ -d /etc/cmk ]; do sleep 1; done; /cmk/cmk.py discover"
+ volumeMounts:
+ - mountPath: /proc
+ name: host-proc
+ readOnly: false
+ - mountPath: /etc/cmk
+ name: cmk-conf-dir
+ readOnly: false
+ - mountPath: /opt/bin
+ name: cmk-install-dir
+ readOnly: false
+ - name: install
+ image: localhost:5000/cmk:v1.4.1
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CMK_PROC_FS
+ value: /proc
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ command: ["/bin/bash", "-c"]
+ args:
+ - "/cmk/cmk.py install"
+ volumeMounts:
+ - mountPath: /proc
+ name: host-proc
+ readOnly: false
+ - mountPath: /etc/cmk
+ name: cmk-conf-dir
+ - mountPath: /opt/bin
+ name: cmk-install-dir
+# restartPolicy: Never
+ volumes:
+ - hostPath:
+ path: /proc
+# type: ""
+ name: host-proc
+ - hostPath:
+ path: /etc/cmk
+# type: ""
+ name: cmk-conf-dir
+ - hostPath:
+ path: /opt/bin
+# type: ""
+ name: cmk-install-dir
+
diff --git a/kud/deployment_infra/playbooks/configure-cmk.yml b/kud/deployment_infra/playbooks/configure-cmk.yml
new file mode 100644
index 00000000..cd2fb50e
--- /dev/null
+++ b/kud/deployment_infra/playbooks/configure-cmk.yml
@@ -0,0 +1,107 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- import_playbook: preconfigure-cmk.yml
+- hosts: localhost
+ pre_tasks:
+ - name: Load kud variables
+ include_vars:
+ file: kud-vars.yml
+ vars:
+ cmk_install_host: '{{ inventory_hostname }}'
+
+ tasks:
+ - name: build list of CMK nodes to untaint
+ set_fact:
+ cmk_hosts_list: "{{ groups['kube-node'] }}"
+
+ - name: generate CMK install yaml file
+ command: "cp {{ playbook_dir }}/../images/cmk.yaml {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+
+ - name: customize CMK install yaml file per runtime env
+ lineinfile:
+ dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+ insertafter: "nodeSelectorTerms:"
+ line: " - matchFields:\n - key: metadata.name\n operator: In\n values:\n - '{{ item }}'"
+ register: cus_result
+ with_items:
+ - "{{ cmk_hosts_list }}"
+
+ - name: prepare CMK CPU cores per config file
+ replace:
+ dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+ regexp: '{{ item.pool }}=0'
+ replace: '{{ item.pool }}={{ item.cores }}'
+ with_items:
+ - { pool: 'num-shared-cores', cores: '{{ cmk_shared_num_cores }}' }
+ - { pool: 'num-exclusive-cores', cores: '{{ cmk_exclusive_num_cores }}' }
+
+ - name: install CMK components
+ command: "/usr/local/bin/kubectl create -f {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+
+ - name: wait for all cmk daemonset pods to be running
+ shell: kubectl get pods -n {{ cmk_namespace }} -l name=cmk -o jsonpath={.items[*].status.phase}
+ register: kubectl_cmk_running
+ until: "['Running'] == kubectl_cmk_running.stdout.split() | unique"
+ retries: 50
+ delay: 5
+ failed_when: false
+
+ - name: create a script to check CMK setup
+ copy:
+ dest: "./cmk-check.sh"
+ content: |
+ #!/bin/bash
+ echo
+ echo "waiting for cmk-nodereport effective"
+ status=0
+ while [ $status -ne 1 ]; do
+ status=$(kubectl get cmk-nodereport | grep ENV | wc -l)
+ sleep 1
+ echo not found
+ done
+ echo "cmk-nodereport is effective"
+
+ - name: judge the runtime environment
+ set_fact:
+ cmk_runtime_env: "{{ groups['kube-node'][0] }}"
+ - debug:
+ var: cmk_runtime_env
+
+ - name: prepare cmk check file
+ replace:
+ dest: "./cmk-check.sh"
+ regexp: 'ENV'
+ replace: '{{ cmk_runtime_env }}'
+
+ - name: Changing perm of "sh", adding "+x"
+ shell: "chmod +x cmk-check.sh"
+ args:
+ warn: false
+
+ - name: Run the script and re-evaluate the variable.
+ command: "./cmk-check.sh"
+
+ - name: Clean the script and folder.
+ file:
+ path: ./cmk-check.sh
+ state: absent
+
+ - name: untaint nodes
+ command: kubectl taint node "{{ item }}" cmk-
+ failed_when: false
+ register: untaint_result
+ changed_when: "untaint_result.rc == 0"
+ when:
+ - cmk_untaint_required
+ with_items:
+ - "{{ cmk_hosts_list }}"
+ - debug:
+ var: untaint_result
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 0fdfafeb..77bf9224 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -40,6 +40,22 @@ istio_source_type: "tarball"
istio_version: 1.0.3
istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz"
+# Intel CPU Manager for Kubernetes
+cmk_enabled: true
+cmk_namespace: kube-system
+cmk_use_all_hosts: false # 'true' will deploy CMK on the master nodes too
+cmk_untaint_nodes: [compute01, compute02]
+cmk_shared_num_cores: 1 # number of CPU cores to be assigned to the "shared" pool on each of the nodes
+cmk_exclusive_num_cores: 2 # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes
+cmk_git_url: "https://github.com/intel/CPU-Manager-for-Kubernetes.git"
+cmk_version: "v1.4.1"
+cmk_dir: "/tmp/cmk"
+registry_local_address: "localhost:5000"
+cmk_pkgs: make,jq
+cmk_untaint_required: true
+#cmk_shared_mode: packed # choose between: packed, spread, default: packed
+#cmk_exclusive_mode: packed # choose between: packed, spread, default: packed
+
go_version: '1.12.5'
kubespray_version: 2.10.4
helm_client_version: 2.13.1
diff --git a/kud/deployment_infra/playbooks/preconfigure-cmk.yml b/kud/deployment_infra/playbooks/preconfigure-cmk.yml
new file mode 100644
index 00000000..7aab4e2e
--- /dev/null
+++ b/kud/deployment_infra/playbooks/preconfigure-cmk.yml
@@ -0,0 +1,62 @@
+---
+- hosts: kube-node
+ become: yes
+ pre_tasks:
+ - name: Load kud variables
+ include_vars:
+ file: kud-vars.yml
+ tasks:
+ - name: install cmk required packges
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items: "{{ cmk_pkgs }}"
+
+ - name: clean CMK directory
+ file:
+ path: "{{ cmk_dir }}"
+ state: absent
+
+ - name: create CMK directory
+ file:
+ path: "{{ cmk_dir }}"
+ state: directory
+
+ - name: clone CMK repository
+ command: git clone {{ cmk_git_url }} -b {{ cmk_version }}
+ args:
+ chdir: "{{ cmk_dir }}"
+
+ - name: read current CMK version
+ command: echo v1.4.1
+ args:
+ chdir: "{{ cmk_dir }}"
+ register: cmk_img_version
+
+ - name: build CMK image
+ command: make
+ args:
+ chdir: "{{ cmk_dir }}/CPU-Manager-for-Kubernetes"
+
+ - name: tag CMK image
+ command: docker tag cmk:{{ cmk_img_version.stdout }} {{ registry_local_address }}/cmk:{{ cmk_img_version.stdout }}
+
+ - name: build list of CMK hosts
+ set_fact:
+ cmk_hosts_list: "{{ groups['kube-node'] | join(',') }}"
+ when:
+ - cmk_use_all_hosts != true
+ - (cmk_hosts_list is undefined) or (cmk_hosts_list | length == 0)
+
+- hosts: kube-master[0]
+ become: yes
+ pre_tasks:
+ - name: Load kud variables
+ include_vars:
+ file: kud-vars.yml
+ tasks:
+ - name: install cmk required packges
+ package:
+ name: "{{ item }}"
+ state: present
+ with_items: "{{ cmk_pkgs }}"