add cmk in KuD

- deploy cmk related pods
- untaint compute nodes if necessary
- run cmk unit tests: allocate CPUs from exclusive and shared pools
- deploy a testing nginx pod along with cmk testing pods
- preset 1/2 CPUs for shared/exlusive pools to fit CI server machines
  users can adjust the parameters to meet their own requirements

Test Results:
  - many rounds of vagrant/5 VMs(controller01/02/03 and compute01/02)
    based test are all OK
  - 14 rounds tests on my local server (S2600WFQ (36C/72T) )and
    PC(HP Z228 (4C/4T)) with all-in-one bare metal deployment are all OK
  - CI(a 4C/4T machine) results of latest patch set also show that the
    test of bare metal deployment is OK
  - NOTE: both my local test and CI use the same testing method of calling
          aio.sh after applying the latest patch set.

Change-Id: I046a4a63b94f92f23347ab76c21a661521e01119
Issue-ID: MULTICLOUD-879
Signed-off-by: Liang Ding <liang.ding@intel.com>

3 files changed, 185 insertions, 0 deletions

diff --git a/kud/deployment_infra/playbooks/configure-cmk.yml b/kud/deployment_infra/playbooks/configure-cmk.yml
new file mode 100644
index 00000000..cd2fb50e
--- /dev/null
+++ b/kud/deployment_infra/playbooks/configure-cmk.yml
@@ -0,0 +1,107 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- import_playbook: preconfigure-cmk.yml
+- hosts: localhost
+  pre_tasks:
+    - name: Load kud variables
+      include_vars:
+        file: kud-vars.yml
+  vars:
+    cmk_install_host: '{{ inventory_hostname }}'
+
+  tasks:
+    - name: build list of CMK nodes to untaint
+      set_fact:
+        cmk_hosts_list: "{{ groups['kube-node'] }}"
+
+    - name: generate CMK install yaml file
+      command: "cp {{ playbook_dir }}/../images/cmk.yaml {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+
+    - name: customize CMK install yaml file per runtime env
+      lineinfile:
+        dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+        insertafter: "nodeSelectorTerms:"
+        line: "            - matchFields:\n              - key: metadata.name\n                operator: In\n                values:\n                - '{{ item }}'"
+      register: cus_result
+      with_items:
+        - "{{ cmk_hosts_list }}"
+
+    - name: prepare CMK CPU cores per config file
+      replace:
+        dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+        regexp: '{{ item.pool }}=0'
+        replace: '{{ item.pool }}={{ item.cores }}'
+      with_items:
+        - { pool: 'num-shared-cores', cores: '{{ cmk_shared_num_cores }}' }
+        - { pool: 'num-exclusive-cores', cores: '{{ cmk_exclusive_num_cores }}' }
+
+    - name: install CMK components
+      command: "/usr/local/bin/kubectl create -f {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml"
+
+    - name: wait for all cmk daemonset pods to be running
+      shell: kubectl get pods -n {{ cmk_namespace }} -l name=cmk -o jsonpath={.items[*].status.phase}
+      register: kubectl_cmk_running
+      until: "['Running'] == kubectl_cmk_running.stdout.split() | unique"
+      retries: 50
+      delay: 5
+      failed_when: false
+
+    - name: create a script to check CMK setup
+      copy:
+        dest: "./cmk-check.sh"
+        content: |
+            #!/bin/bash
+            echo
+            echo "waiting for cmk-nodereport effective"
+              status=0
+              while [ $status -ne 1 ]; do
+                 status=$(kubectl get cmk-nodereport | grep ENV | wc -l)
+                 sleep 1
+                 echo not found
+              done
+            echo "cmk-nodereport is effective"
+
+    - name: judge the runtime environment
+      set_fact:
+        cmk_runtime_env: "{{ groups['kube-node'][0] }}"
+    - debug:
+        var: cmk_runtime_env
+
+    - name: prepare cmk check file
+      replace:
+        dest: "./cmk-check.sh"
+        regexp: 'ENV'
+        replace: '{{ cmk_runtime_env }}'
+
+    - name: Changing perm of "sh", adding "+x"
+      shell: "chmod +x cmk-check.sh"
+      args:
+        warn: false
+
+    - name: Run the script and re-evaluate the variable.
+      command: "./cmk-check.sh"
+
+    - name: Clean the script and folder.
+      file:
+        path: ./cmk-check.sh
+        state: absent
+
+    - name: untaint nodes
+      command: kubectl taint node "{{ item }}" cmk-
+      failed_when: false
+      register: untaint_result
+      changed_when: "untaint_result.rc == 0"
+      when:
+        - cmk_untaint_required
+      with_items:
+        - "{{ cmk_hosts_list }}"
+    - debug:
+        var: untaint_result
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 0fdfafeb..77bf9224 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -40,6 +40,22 @@ istio_source_type: "tarball"
 istio_version: 1.0.3
 istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz"
 
+# Intel CPU Manager for Kubernetes
+cmk_enabled: true
+cmk_namespace: kube-system
+cmk_use_all_hosts: false # 'true' will deploy CMK on the master nodes too
+cmk_untaint_nodes: [compute01, compute02]
+cmk_shared_num_cores: 1 # number of CPU cores to be assigned to the "shared" pool on each of the nodes
+cmk_exclusive_num_cores: 2 # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes
+cmk_git_url: "https://github.com/intel/CPU-Manager-for-Kubernetes.git"
+cmk_version: "v1.4.1"
+cmk_dir: "/tmp/cmk"
+registry_local_address: "localhost:5000"
+cmk_pkgs: make,jq
+cmk_untaint_required: true
+#cmk_shared_mode: packed # choose between: packed, spread, default: packed
+#cmk_exclusive_mode: packed # choose between: packed, spread, default: packed
+
 go_version: '1.12.5'
 kubespray_version: 2.10.4
 helm_client_version: 2.13.1
diff --git a/kud/deployment_infra/playbooks/preconfigure-cmk.yml b/kud/deployment_infra/playbooks/preconfigure-cmk.yml
new file mode 100644
index 00000000..7aab4e2e
--- /dev/null
+++ b/kud/deployment_infra/playbooks/preconfigure-cmk.yml
@@ -0,0 +1,62 @@
+---
+- hosts: kube-node
+  become: yes
+  pre_tasks:
+    - name: Load kud variables
+      include_vars:
+        file: kud-vars.yml
+  tasks:
+    - name: install cmk required packges
+      package:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ cmk_pkgs }}"
+
+    - name: clean CMK directory
+      file:
+        path: "{{ cmk_dir }}"
+        state: absent
+
+    - name: create CMK directory
+      file:
+        path: "{{ cmk_dir }}"
+        state: directory
+
+    - name: clone CMK repository
+      command: git clone {{ cmk_git_url }} -b {{ cmk_version }}
+      args:
+        chdir: "{{ cmk_dir }}"
+
+    - name: read current CMK version
+      command: echo v1.4.1
+      args:
+        chdir: "{{ cmk_dir }}"
+      register: cmk_img_version
+
+    - name: build CMK image
+      command: make
+      args:
+        chdir: "{{ cmk_dir }}/CPU-Manager-for-Kubernetes"
+
+    - name: tag CMK image
+      command: docker tag cmk:{{ cmk_img_version.stdout }} {{ registry_local_address }}/cmk:{{ cmk_img_version.stdout }}
+
+    - name: build list of CMK hosts
+      set_fact:
+        cmk_hosts_list: "{{ groups['kube-node'] | join(',') }}"
+      when:
+        - cmk_use_all_hosts != true
+        - (cmk_hosts_list is undefined) or (cmk_hosts_list | length == 0)
+
+- hosts: kube-master[0]
+  become: yes
+  pre_tasks:
+    - name: Load kud variables
+      include_vars:
+        file: kud-vars.yml
+  tasks:
+    - name: install cmk required packges
+      package:
+        name: "{{ item }}"
+        state: present
+      with_items: "{{ cmk_pkgs }}"