From cc10b9aff3fd16df62c0655ec1626624ad2e2fc0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 30 Oct 2019 23:47:35 -0700 Subject: add cmk in KuD - deploy cmk related pods - untaint compute nodes if necessary - run cmk unit tests: allocate CPUs from exclusive and shared pools - deploy a testing nginx pod along with cmk testing pods - preset 1/2 CPUs for shared/exlusive pools to fit CI server machines users can adjust the parameters to meet their own requirements Test Results: - many rounds of vagrant/5 VMs(controller01/02/03 and compute01/02) based test are all OK - 14 rounds tests on my local server (S2600WFQ (36C/72T) )and PC(HP Z228 (4C/4T)) with all-in-one bare metal deployment are all OK - CI(a 4C/4T machine) results of latest patch set also show that the test of bare metal deployment is OK - NOTE: both my local test and CI use the same testing method of calling aio.sh after applying the latest patch set. Change-Id: I046a4a63b94f92f23347ab76c21a661521e01119 Issue-ID: MULTICLOUD-879 Signed-off-by: Liang Ding --- kud/deployment_infra/playbooks/configure-cmk.yml | 107 +++++++++++++++++++++ kud/deployment_infra/playbooks/kud-vars.yml | 16 +++ .../playbooks/preconfigure-cmk.yml | 62 ++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 kud/deployment_infra/playbooks/configure-cmk.yml create mode 100644 kud/deployment_infra/playbooks/preconfigure-cmk.yml (limited to 'kud/deployment_infra/playbooks') diff --git a/kud/deployment_infra/playbooks/configure-cmk.yml b/kud/deployment_infra/playbooks/configure-cmk.yml new file mode 100644 index 00000000..cd2fb50e --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-cmk.yml @@ -0,0 +1,107 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- import_playbook: preconfigure-cmk.yml +- hosts: localhost + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + vars: + cmk_install_host: '{{ inventory_hostname }}' + + tasks: + - name: build list of CMK nodes to untaint + set_fact: + cmk_hosts_list: "{{ groups['kube-node'] }}" + + - name: generate CMK install yaml file + command: "cp {{ playbook_dir }}/../images/cmk.yaml {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml" + + - name: customize CMK install yaml file per runtime env + lineinfile: + dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml" + insertafter: "nodeSelectorTerms:" + line: " - matchFields:\n - key: metadata.name\n operator: In\n values:\n - '{{ item }}'" + register: cus_result + with_items: + - "{{ cmk_hosts_list }}" + + - name: prepare CMK CPU cores per config file + replace: + dest: "{{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml" + regexp: '{{ item.pool }}=0' + replace: '{{ item.pool }}={{ item.cores }}' + with_items: + - { pool: 'num-shared-cores', cores: '{{ cmk_shared_num_cores }}' } + - { pool: 'num-exclusive-cores', cores: '{{ cmk_exclusive_num_cores }}' } + + - name: install CMK components + command: "/usr/local/bin/kubectl create -f {{ playbook_dir }}/../images/cmk-{{ cmk_install_host }}.yaml" + + - name: wait for all cmk daemonset pods to be running + shell: kubectl get pods -n {{ cmk_namespace }} -l name=cmk -o jsonpath={.items[*].status.phase} + register: kubectl_cmk_running + until: "['Running'] == kubectl_cmk_running.stdout.split() | unique" + retries: 50 + delay: 5 + failed_when: false + + - name: create a script to check CMK setup + copy: + dest: "./cmk-check.sh" + content: | + #!/bin/bash + echo + echo "waiting for cmk-nodereport effective" + status=0 + while [ $status -ne 1 ]; do + status=$(kubectl get cmk-nodereport | grep ENV | wc -l) + sleep 1 + echo not found + done + echo "cmk-nodereport is effective" + + - name: judge the runtime environment + set_fact: + cmk_runtime_env: "{{ groups['kube-node'][0] }}" + - debug: + var: cmk_runtime_env + + - name: prepare cmk check file + replace: + dest: "./cmk-check.sh" + regexp: 'ENV' + replace: '{{ cmk_runtime_env }}' + + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x cmk-check.sh" + args: + warn: false + + - name: Run the script and re-evaluate the variable. + command: "./cmk-check.sh" + + - name: Clean the script and folder. + file: + path: ./cmk-check.sh + state: absent + + - name: untaint nodes + command: kubectl taint node "{{ item }}" cmk- + failed_when: false + register: untaint_result + changed_when: "untaint_result.rc == 0" + when: + - cmk_untaint_required + with_items: + - "{{ cmk_hosts_list }}" + - debug: + var: untaint_result diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 0fdfafeb..77bf9224 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -40,6 +40,22 @@ istio_source_type: "tarball" istio_version: 1.0.3 istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz" +# Intel CPU Manager for Kubernetes +cmk_enabled: true +cmk_namespace: kube-system +cmk_use_all_hosts: false # 'true' will deploy CMK on the master nodes too +cmk_untaint_nodes: [compute01, compute02] +cmk_shared_num_cores: 1 # number of CPU cores to be assigned to the "shared" pool on each of the nodes +cmk_exclusive_num_cores: 2 # number of CPU cores to be assigned to the "exclusive" pool on each of the nodes +cmk_git_url: "https://github.com/intel/CPU-Manager-for-Kubernetes.git" +cmk_version: "v1.4.1" +cmk_dir: "/tmp/cmk" +registry_local_address: "localhost:5000" +cmk_pkgs: make,jq +cmk_untaint_required: true +#cmk_shared_mode: packed # choose between: packed, spread, default: packed +#cmk_exclusive_mode: packed # choose between: packed, spread, default: packed + go_version: '1.12.5' kubespray_version: 2.10.4 helm_client_version: 2.13.1 diff --git a/kud/deployment_infra/playbooks/preconfigure-cmk.yml b/kud/deployment_infra/playbooks/preconfigure-cmk.yml new file mode 100644 index 00000000..7aab4e2e --- /dev/null +++ b/kud/deployment_infra/playbooks/preconfigure-cmk.yml @@ -0,0 +1,62 @@ +--- +- hosts: kube-node + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: install cmk required packges + package: + name: "{{ item }}" + state: present + with_items: "{{ cmk_pkgs }}" + + - name: clean CMK directory + file: + path: "{{ cmk_dir }}" + state: absent + + - name: create CMK directory + file: + path: "{{ cmk_dir }}" + state: directory + + - name: clone CMK repository + command: git clone {{ cmk_git_url }} -b {{ cmk_version }} + args: + chdir: "{{ cmk_dir }}" + + - name: read current CMK version + command: echo v1.4.1 + args: + chdir: "{{ cmk_dir }}" + register: cmk_img_version + + - name: build CMK image + command: make + args: + chdir: "{{ cmk_dir }}/CPU-Manager-for-Kubernetes" + + - name: tag CMK image + command: docker tag cmk:{{ cmk_img_version.stdout }} {{ registry_local_address }}/cmk:{{ cmk_img_version.stdout }} + + - name: build list of CMK hosts + set_fact: + cmk_hosts_list: "{{ groups['kube-node'] | join(',') }}" + when: + - cmk_use_all_hosts != true + - (cmk_hosts_list is undefined) or (cmk_hosts_list | length == 0) + +- hosts: kube-master[0] + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + tasks: + - name: install cmk required packges + package: + name: "{{ item }}" + state: present + with_items: "{{ cmk_pkgs }}" -- cgit 1.2.3-korg