From cc302424c0b129267235157640be28b45597fef2 Mon Sep 17 00:00:00 2001 From: Todd Malsbary Date: Fri, 26 Mar 2021 14:17:49 -0700 Subject: EMCO deploy of addons An example is provided with instructions on how to install the addons with emcoctl. Addtionally, the containerized installer will populate /opt/kud/addons and /opt/kud/multi-cluster/$CLUSTER_NAME/artifacts with the files and instructions necessary as well. Issue-ID: MULTICLOUD-1324 Signed-off-by: Todd Malsbary Change-Id: I74de1c9d18a0aaec4a96e38684ec80f00ab0b940 --- .../containerized/addons/README.md.tmpl | 45 ++++++++++++++++ .../addons/values-resources.yaml.tmpl | 19 +++++++ .../containerized/addons/values.yaml.tmpl | 24 +++++++++ kud/hosting_providers/containerized/installer.sh | 62 +++++++++++++++++----- kud/hosting_providers/vagrant/installer.sh | 4 +- 5 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 kud/hosting_providers/containerized/addons/README.md.tmpl create mode 100644 kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl create mode 100644 kud/hosting_providers/containerized/addons/values.yaml.tmpl (limited to 'kud/hosting_providers') diff --git a/kud/hosting_providers/containerized/addons/README.md.tmpl b/kud/hosting_providers/containerized/addons/README.md.tmpl new file mode 100644 index 00000000..8ab16104 --- /dev/null +++ b/kud/hosting_providers/containerized/addons/README.md.tmpl @@ -0,0 +1,45 @@ +# Installing KUD addons with emcoctl + +1. Customize values.yaml and values-resources.yaml as needed + +To create a customized profile for a specific addon, edit the profile +as needed, and then (for example, cpu-manager): + +``` + tar -czf /opt/kud/multi-cluster/addons/cpu-manager.tar.gz -C /opt/kud/multi-cluster/addons/cpu-manager/helm . + tar -czf /opt/kud/multi-cluster/addons/collectd_profile.tar.gz -C /opt/kud/multi-cluster/addons/cpu-manager/profile . +``` + +2. Create prerequisites to deploy addons + +Apply prerequisites.yaml. This step is optional. If there are +existing resources in the cluster, it is sufficient to customize +values.yaml with the values of those resources. The supplied +prequisites.yaml creates controllers, one project, one cluster, and +one logical cloud. + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f prerequisites.yaml -v values.yaml\` + +3. Deploy addons + +Apply addons.yaml. This deploys the addons listed in the \`Addons\` +value in values.yaml. + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values.yaml\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values-resources.yaml\` + +# Uninstalling KUD addons with emcoctl + +1. Delete addons + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values-resources.yaml\` + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values.yaml\` + +2. Cleanup prerequisites + + \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f prerequisites.yaml -v values.yaml\` + +#### NOTE: Known issue: deletion of the resources fails sometimes as +some resources can't be deleted before others are deleted. This can +happen due to timing issue. In that case try deleting again and the +deletion should succeed. diff --git a/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl b/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl new file mode 100644 index 00000000..ed568238 --- /dev/null +++ b/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl @@ -0,0 +1,19 @@ +HostIP: ${HOST_IP} +KubeConfig: ${KUBE_PATH} +PackagesPath: ${PACKAGES_PATH} +ProjectName: proj1 +RsyncPort: 30441 +GacPort: 30493 +OvnPort: 30473 +DtcPort: 30483 +ClusterProvider: provider1 +Cluster1: cluster1 +ClusterLabel: edge-cluster +LogicalCloud: default +Apps: +- sriov-network +CompositeApp: addon-resources +CompositeProfile: addon-resources-profile +DeploymentIntentGroup: addon-resources-deployment-intent-group +DeploymentIntent: addon-resources-deployment-intent +GenericPlacementIntent: addon-resources-placement-intent diff --git a/kud/hosting_providers/containerized/addons/values.yaml.tmpl b/kud/hosting_providers/containerized/addons/values.yaml.tmpl new file mode 100644 index 00000000..62936beb --- /dev/null +++ b/kud/hosting_providers/containerized/addons/values.yaml.tmpl @@ -0,0 +1,24 @@ +HostIP: ${HOST_IP} +KubeConfig: ${KUBE_PATH} +PackagesPath: ${PACKAGES_PATH} +ProjectName: proj1 +RsyncPort: 30441 +GacPort: 30493 +OvnPort: 30473 +DtcPort: 30483 +ClusterProvider: provider1 +Cluster1: cluster1 +ClusterLabel: edge-cluster +LogicalCloud: default +Apps: +- multus-cni +- ovn4nfv +- node-feature-discovery +- sriov-network-operator +- qat-device-plugin +- cpu-manager +CompositeApp: addons +CompositeProfile: addons-profile +DeploymentIntentGroup: addons-deployment-intent-group +DeploymentIntent: addons-deployment-intent +GenericPlacementIntent: addons-placement-intent diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index eb30a23e..74c031dc 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -14,6 +14,7 @@ set -o pipefail set -ex INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +KUD_ADDONS="" function install_prerequisites { #install package for docker images @@ -22,7 +23,7 @@ function install_prerequisites { find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true apt-get update apt-get install -y curl vim wget git \ - software-properties-common python-pip sudo + software-properties-common python-pip sudo gettext-base add-apt-repository -y ppa:longsleep/golang-backports apt-get update apt-get install -y golang-go rsync @@ -119,7 +120,7 @@ function install_addons { tee $cluster_log/setup-kud.log # The order of KUD_ADDONS is important: some plugins (sriov, qat) # require nfd to be enabled. - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do + for addon in $KUD_ADDONS $plugins_name; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | \ @@ -129,7 +130,7 @@ function install_addons { echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then failed_kud_tests="" - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do + for addon in $KUD_ADDONS $plugins_name; do pushd $kud_tests bash ${addon}.sh || failed_kud_tests="${failed_kud_tests} ${addon}" case $addon in @@ -141,9 +142,10 @@ function install_addons { ;; "emco" ) echo "Test the emco plugin installation" - for functional_test in plugin_fw_v2; do - bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}" - done + # TODO plugin_fw_v2 requires virtlet and a patched multus to succeed + # for functional_test in plugin_fw_v2; do + # bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}" + # done ;; esac popd @@ -156,6 +158,45 @@ function install_addons { echo "Add-ons deployment complete..." } +function master_ip { + kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}' +} + +# Copy installation artifacts to be usable in host running Ansible +function install_host_artifacts { + local -r cluster_name=$1 + local -r host_dir="/opt/kud/multi-cluster" + local -r host_addons_dir="${host_dir}/addons" + local -r host_artifacts_dir="${host_dir}/${cluster_name}/artifacts" + + for addon in cpu-manager multus-cni node-feature-discovery ovn4nfv qat-device-plugin sriov-network sriov-network-operator; do + mkdir -p ${host_addons_dir}/${addon}/{helm,profile} + cp -r ${kud_infra_folder}/helm/${addon} ${host_addons_dir}/${addon}/helm + cp -r ${kud_infra_folder}/profiles/${addon}/* ${host_addons_dir}/${addon}/profile + tar -czf ${host_addons_dir}/${addon}.tar.gz -C ${host_addons_dir}/${addon}/helm . + tar -czf ${host_addons_dir}/${addon}_profile.tar.gz -C ${host_addons_dir}/${addon}/profile . + done + + mkdir -p ${host_addons_dir}/tests + for test in _common _common_test _functions multus ovn4nfv nfd sriov-network qat cmk; do + cp ${kud_tests}/${test}.sh ${host_addons_dir}/tests + done + + mkdir -p ${host_artifacts_dir} + cp -rf ${kud_inventory_folder}/artifacts/* ${host_artifacts_dir} + + mkdir -p ${host_artifacts_dir}/addons + cp ${kud_infra_folder}/emco/examples/prerequisites.yaml ${host_artifacts_dir}/addons + cp ${kud_infra_folder}/emco/composite-app.yaml ${host_artifacts_dir}/addons + for template in addons/*.tmpl; do + CLUSTER_NAME="${cluster_name}" \ + HOST_IP="$(master_ip)" \ + KUBE_PATH="${host_artifacts_dir}/admin.conf" \ + PACKAGES_PATH="${host_addons_dir}" \ + envsubst <${template} >${host_artifacts_dir}/${template%.tmpl} + done +} + # _print_kubernetes_info() - Prints the login Kubernetes information function _print_kubernetes_info { if ! $(kubectl version &>/dev/null); then @@ -169,11 +210,8 @@ function _print_kubernetes_info { KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" \ kubectl -n kube-system edit service kubernetes-dashboard - master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ - awk -F ":" '{print $2}') - printf "Kubernetes Info\n===============\n" > $k8s_info_file - echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Dashboard URL: https://$(master_ip):$node_port" >> $k8s_info_file echo "Admin user: kube" >> $k8s_info_file echo "Admin password: secret" >> $k8s_info_file } @@ -216,9 +254,7 @@ function install_cluster { fi echo "installed the addons" - # Copy installation artifacts to be usable in host running Ansible - cp -rf $kud_inventory_folder/artifacts \ - /opt/kud/multi-cluster/$cluster_name/ + install_host_artifacts $1 _print_kubernetes_info } diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 9be77a1f..bc2e91ae 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -213,10 +213,10 @@ function _print_kubernetes_info { KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard - master_ip=$(kubectl cluster-info | grep "Kubernetes master" | awk -F ":" '{print $2}') + master_ip=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}') printf "Kubernetes Info\n===============\n" > $k8s_info_file - echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Dashboard URL: https://$master_ip:$node_port" >> $k8s_info_file echo "Admin user: kube" >> $k8s_info_file echo "Admin password: secret" >> $k8s_info_file } -- cgit 1.2.3-korg