From cc10b9aff3fd16df62c0655ec1626624ad2e2fc0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 30 Oct 2019 23:47:35 -0700 Subject: add cmk in KuD - deploy cmk related pods - untaint compute nodes if necessary - run cmk unit tests: allocate CPUs from exclusive and shared pools - deploy a testing nginx pod along with cmk testing pods - preset 1/2 CPUs for shared/exlusive pools to fit CI server machines users can adjust the parameters to meet their own requirements Test Results: - many rounds of vagrant/5 VMs(controller01/02/03 and compute01/02) based test are all OK - 14 rounds tests on my local server (S2600WFQ (36C/72T) )and PC(HP Z228 (4C/4T)) with all-in-one bare metal deployment are all OK - CI(a 4C/4T machine) results of latest patch set also show that the test of bare metal deployment is OK - NOTE: both my local test and CI use the same testing method of calling aio.sh after applying the latest patch set. Change-Id: I046a4a63b94f92f23347ab76c21a661521e01119 Issue-ID: MULTICLOUD-879 Signed-off-by: Liang Ding --- kud/hosting_providers/baremetal/README.md | 4 ++++ kud/hosting_providers/baremetal/aio.sh | 3 +++ kud/hosting_providers/containerized/installer.sh | 4 ++-- kud/hosting_providers/vagrant/Vagrantfile | 24 +++++++++++++++++----- kud/hosting_providers/vagrant/config/default.yml | 6 ++++-- .../vagrant/config/samples/pdf.yml.aio | 1 + .../vagrant/config/samples/pdf.yml.mini | 2 ++ kud/hosting_providers/vagrant/installer.sh | 4 ++-- 8 files changed, 37 insertions(+), 11 deletions(-) (limited to 'kud/hosting_providers') diff --git a/kud/hosting_providers/baremetal/README.md b/kud/hosting_providers/baremetal/README.md index 4f81d7b5..5e1edf79 100644 --- a/kud/hosting_providers/baremetal/README.md +++ b/kud/hosting_providers/baremetal/README.md @@ -15,6 +15,10 @@ ansible playbooks allow to provision a deployment on Baremetal. The [installer](installer.sh) bash script contains the minimal Ubuntu instructions required for running this project. +NOTE: for cmk bare metal deployment, preset 1/2 CPUs for + shared/exlusive pools respectively to fit CI server machines + users can adjust the parameters to meet their own requirements. + ## License Apache-2.0 diff --git a/kud/hosting_providers/baremetal/aio.sh b/kud/hosting_providers/baremetal/aio.sh index c9fac098..6a304141 100755 --- a/kud/hosting_providers/baremetal/aio.sh +++ b/kud/hosting_providers/baremetal/aio.sh @@ -41,6 +41,9 @@ localhost [virtlet] localhost +[cmk] +localhost + [k8s-cluster:children] kube-node kube-master diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 8739ca23..afea0b5a 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -119,7 +119,7 @@ function install_addons { ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-kud.yml | \ tee $cluster_log/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-${addon}.yml | \ @@ -128,7 +128,7 @@ function install_addons { echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov cmk $plugins_name}; do pushd $kud_tests bash ${addon}.sh popd diff --git a/kud/hosting_providers/vagrant/Vagrantfile b/kud/hosting_providers/vagrant/Vagrantfile index eb5e5cdc..bcaa9d0c 100644 --- a/kud/hosting_providers/vagrant/Vagrantfile +++ b/kud/hosting_providers/vagrant/Vagrantfile @@ -27,7 +27,7 @@ File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_fi nodes.each do |node| inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") end - ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet'].each do|group| + ['kube-master', 'kube-node', 'etcd', 'ovn-central', 'ovn-controller', 'virtlet', 'cmk'].each do|group| inventory_file.puts("\n[#{group}]") nodes.each do |node| if node['roles'].include?("#{group}") @@ -76,8 +76,24 @@ Vagrant.configure("2") do |config| v.random_hostname = true end + sync_type = "virtualbox" + if provider == :libvirt + sync_type = "nfs" + end + nodes.each do |node| config.vm.define node['name'] do |nodeconfig| + if node['roles'].include?("kube-master") + nodeconfig.vm.synced_folder '../../../', '/home/vagrant/multicloud-k8s/', type: sync_type + end + if node['roles'].include?("kube-node") + nodeconfig.vm.provision 'shell', privileged: false do |sh| + sh.inline = <<-SHELL + sudo sed -i 's:GRUB_CMDLINE_LINUX=.*:GRUB_CMDLINE_LINUX="isolcpus=0-7":' /etc/default/grub + sudo update-grub + SHELL + end + end nodeconfig.vm.hostname = node['name'] nodeconfig.vm.network :private_network, :ip => node['ip'], :type => :static nodeconfig.vm.provider 'virtualbox' do |v| @@ -111,10 +127,7 @@ Vagrant.configure("2") do |config| end end end - sync_type = "virtualbox" - if provider == :libvirt - sync_type = "nfs" - end + config.vm.define :installer, primary: true, autostart: false do |installer| installer.vm.hostname = "multicloud" installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static @@ -126,6 +139,7 @@ Vagrant.configure("2") do |config| cp /home/vagrant/multicloud-k8s/kud/hosting_providers/vagrant/insecure_keys/key /home/vagrant/.ssh/id_rsa chown vagrant /home/vagrant/.ssh/id_rsa chmod 400 /home/vagrant/.ssh/id_rsa + sudo apt install jq -y cd /home/vagrant/multicloud-k8s/kud/hosting_providers/vagrant/ && ./installer.sh | tee kud_installer.log SHELL end diff --git a/kud/hosting_providers/vagrant/config/default.yml b/kud/hosting_providers/vagrant/config/default.yml index 094c3594..242998c4 100644 --- a/kud/hosting_providers/vagrant/config/default.yml +++ b/kud/hosting_providers/vagrant/config/default.yml @@ -44,10 +44,12 @@ - kube-node - ovn-controller - virtlet + - cmk - name: "compute02" ip: "10.10.10.7" - memory: 8192 - cpus: 4 + memory: 32768 + cpus: 16 roles: - kube-node - ovn-controller + - cmk diff --git a/kud/hosting_providers/vagrant/config/samples/pdf.yml.aio b/kud/hosting_providers/vagrant/config/samples/pdf.yml.aio index a87f967e..cd95776f 100644 --- a/kud/hosting_providers/vagrant/config/samples/pdf.yml.aio +++ b/kud/hosting_providers/vagrant/config/samples/pdf.yml.aio @@ -23,3 +23,4 @@ - kube-node - ovn-controller - virtlet + - cmk diff --git a/kud/hosting_providers/vagrant/config/samples/pdf.yml.mini b/kud/hosting_providers/vagrant/config/samples/pdf.yml.mini index d53a4537..258d7799 100644 --- a/kud/hosting_providers/vagrant/config/samples/pdf.yml.mini +++ b/kud/hosting_providers/vagrant/config/samples/pdf.yml.mini @@ -24,6 +24,7 @@ - kube-node - ovn-controller - virtlet + - cmk - name: "minion02" ip: "10.10.10.5" memory: 65536 @@ -31,3 +32,4 @@ roles: - kube-node - ovn-controller + - cmk diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 546d4058..859b49ce 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -155,13 +155,13 @@ function install_addons { _install_ansible sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log done echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk}; do pushd $kud_tests bash ${addon}.sh popd -- cgit 1.2.3-korg