EMCO deploy of addons

An example is provided with instructions on how to install the addons
with emcoctl.  Addtionally, the containerized installer will populate
/opt/kud/addons and /opt/kud/multi-cluster/$CLUSTER_NAME/artifacts
with the files and instructions necessary as well.

Issue-ID: MULTICLOUD-1324
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I74de1c9d18a0aaec4a96e38684ec80f00ab0b940

4 files changed, 137 insertions, 13 deletions

diff --git a/kud/hosting_providers/containerized/addons/README.md.tmpl b/kud/hosting_providers/containerized/addons/README.md.tmpl
new file mode 100644
index 00000000..8ab16104
--- /dev/null
+++ b/kud/hosting_providers/containerized/addons/README.md.tmpl
@@ -0,0 +1,45 @@
+# Installing KUD addons with emcoctl
+
+1. Customize values.yaml and values-resources.yaml as needed
+
+To create a customized profile for a specific addon, edit the profile
+as needed, and then (for example, cpu-manager):
+
+```
+    tar -czf /opt/kud/multi-cluster/addons/cpu-manager.tar.gz -C /opt/kud/multi-cluster/addons/cpu-manager/helm .
+    tar -czf /opt/kud/multi-cluster/addons/collectd_profile.tar.gz -C /opt/kud/multi-cluster/addons/cpu-manager/profile .
+```
+
+2. Create prerequisites to deploy addons
+
+Apply prerequisites.yaml.  This step is optional.  If there are
+existing resources in the cluster, it is sufficient to customize
+values.yaml with the values of those resources.  The supplied
+prequisites.yaml creates controllers, one project, one cluster, and
+one logical cloud.
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f prerequisites.yaml -v values.yaml\`
+
+3. Deploy addons
+
+Apply addons.yaml. This deploys the addons listed in the \`Addons\`
+value in values.yaml.
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values.yaml\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh apply -f composite-app.yaml -v values-resources.yaml\`
+
+# Uninstalling KUD addons with emcoctl
+
+1. Delete addons
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values-resources.yaml\`
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f composite-app.yaml -v values.yaml\`
+
+2. Cleanup prerequisites
+
+    \`$ /opt/kud/multi-cluster/${CLUSTER_NAME}/artifacts/emcoctl.sh delete -f prerequisites.yaml -v values.yaml\`
+
+#### NOTE: Known issue: deletion of the resources fails sometimes as
+some resources can't be deleted before others are deleted. This can
+happen due to timing issue. In that case try deleting again and the
+deletion should succeed.
diff --git a/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl b/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl
new file mode 100644
index 00000000..ed568238
--- /dev/null
+++ b/kud/hosting_providers/containerized/addons/values-resources.yaml.tmpl
@@ -0,0 +1,19 @@
+HostIP: ${HOST_IP}
+KubeConfig: ${KUBE_PATH}
+PackagesPath: ${PACKAGES_PATH}
+ProjectName: proj1
+RsyncPort: 30441
+GacPort: 30493
+OvnPort: 30473
+DtcPort: 30483
+ClusterProvider: provider1
+Cluster1: cluster1
+ClusterLabel: edge-cluster
+LogicalCloud: default
+Apps:
+- sriov-network
+CompositeApp: addon-resources
+CompositeProfile: addon-resources-profile
+DeploymentIntentGroup: addon-resources-deployment-intent-group
+DeploymentIntent: addon-resources-deployment-intent
+GenericPlacementIntent: addon-resources-placement-intent
diff --git a/kud/hosting_providers/containerized/addons/values.yaml.tmpl b/kud/hosting_providers/containerized/addons/values.yaml.tmpl
new file mode 100644
index 00000000..62936beb
--- /dev/null
+++ b/kud/hosting_providers/containerized/addons/values.yaml.tmpl
@@ -0,0 +1,24 @@
+HostIP: ${HOST_IP}
+KubeConfig: ${KUBE_PATH}
+PackagesPath: ${PACKAGES_PATH}
+ProjectName: proj1
+RsyncPort: 30441
+GacPort: 30493
+OvnPort: 30473
+DtcPort: 30483
+ClusterProvider: provider1
+Cluster1: cluster1
+ClusterLabel: edge-cluster
+LogicalCloud: default
+Apps:
+- multus-cni
+- ovn4nfv
+- node-feature-discovery
+- sriov-network-operator
+- qat-device-plugin
+- cpu-manager
+CompositeApp: addons
+CompositeProfile: addons-profile
+DeploymentIntentGroup: addons-deployment-intent-group
+DeploymentIntent: addons-deployment-intent
+GenericPlacementIntent: addons-placement-intent
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index eb30a23e..74c031dc 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -14,6 +14,7 @@ set -o pipefail
 set -ex
 
 INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+KUD_ADDONS=""
 
 function install_prerequisites {
 #install package for docker images
@@ -22,7 +23,7 @@ function install_prerequisites {
     find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true
     apt-get update
     apt-get install -y curl vim wget git \
-        software-properties-common python-pip sudo
+        software-properties-common python-pip sudo gettext-base
     add-apt-repository -y ppa:longsleep/golang-backports
     apt-get update
     apt-get install -y golang-go rsync
@@ -119,7 +120,7 @@ function install_addons {
         tee $cluster_log/setup-kud.log
     # The order of KUD_ADDONS is important: some plugins (sriov, qat)
     # require nfd to be enabled.
-    for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
+    for addon in $KUD_ADDONS $plugins_name; do
         echo "Deploying $addon using configure-$addon.yml playbook.."
         ansible-playbook $verbose -i \
             $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | \
@@ -129,7 +130,7 @@ function install_addons {
     echo "Run the test cases if testing_enabled is set to true."
     if [[ "${testing_enabled}" == "true" ]]; then
         failed_kud_tests=""
-        for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat cmk $plugins_name}; do
+        for addon in $KUD_ADDONS $plugins_name; do
             pushd $kud_tests
             bash ${addon}.sh || failed_kud_tests="${failed_kud_tests} ${addon}"
             case $addon in
@@ -141,9 +142,10 @@ function install_addons {
                     ;;
                 "emco" )
                     echo "Test the emco plugin installation"
-                    for functional_test in plugin_fw_v2; do
-                        bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}"
-                    done
+                    # TODO plugin_fw_v2 requires virtlet and a patched multus to succeed
+                    # for functional_test in plugin_fw_v2; do
+                    #     bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}"
+                    # done
                     ;;
             esac
             popd
@@ -156,6 +158,45 @@ function install_addons {
     echo "Add-ons deployment complete..."
 }
 
+function master_ip {
+    kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}'
+}
+
+# Copy installation artifacts to be usable in host running Ansible
+function install_host_artifacts {
+    local -r cluster_name=$1
+    local -r host_dir="/opt/kud/multi-cluster"
+    local -r host_addons_dir="${host_dir}/addons"
+    local -r host_artifacts_dir="${host_dir}/${cluster_name}/artifacts"
+
+    for addon in cpu-manager multus-cni node-feature-discovery ovn4nfv qat-device-plugin sriov-network sriov-network-operator; do
+        mkdir -p ${host_addons_dir}/${addon}/{helm,profile}
+        cp -r ${kud_infra_folder}/helm/${addon} ${host_addons_dir}/${addon}/helm
+        cp -r ${kud_infra_folder}/profiles/${addon}/* ${host_addons_dir}/${addon}/profile
+        tar -czf ${host_addons_dir}/${addon}.tar.gz -C ${host_addons_dir}/${addon}/helm .
+        tar -czf ${host_addons_dir}/${addon}_profile.tar.gz -C ${host_addons_dir}/${addon}/profile .
+    done
+
+    mkdir -p ${host_addons_dir}/tests
+    for test in _common _common_test _functions multus ovn4nfv nfd sriov-network qat cmk; do
+        cp ${kud_tests}/${test}.sh ${host_addons_dir}/tests
+    done
+
+    mkdir -p ${host_artifacts_dir}
+    cp -rf ${kud_inventory_folder}/artifacts/* ${host_artifacts_dir}
+
+    mkdir -p ${host_artifacts_dir}/addons
+    cp ${kud_infra_folder}/emco/examples/prerequisites.yaml ${host_artifacts_dir}/addons
+    cp ${kud_infra_folder}/emco/composite-app.yaml ${host_artifacts_dir}/addons
+    for template in addons/*.tmpl; do
+        CLUSTER_NAME="${cluster_name}" \
+        HOST_IP="$(master_ip)" \
+        KUBE_PATH="${host_artifacts_dir}/admin.conf" \
+        PACKAGES_PATH="${host_addons_dir}" \
+        envsubst <${template} >${host_artifacts_dir}/${template%.tmpl}
+    done
+}
+
 # _print_kubernetes_info() - Prints the login Kubernetes information
 function _print_kubernetes_info {
     if ! $(kubectl version &>/dev/null); then
@@ -169,11 +210,8 @@ function _print_kubernetes_info {
     KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" \
         kubectl -n kube-system edit service kubernetes-dashboard
 
-    master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \
-        awk -F ":" '{print $2}')
-
     printf "Kubernetes Info\n===============\n" > $k8s_info_file
-    echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file
+    echo "Dashboard URL: https://$(master_ip):$node_port" >> $k8s_info_file
     echo "Admin user: kube" >> $k8s_info_file
     echo "Admin password: secret" >> $k8s_info_file
 }
@@ -216,9 +254,7 @@ function install_cluster {
     fi
     echo "installed the addons"
 
-    # Copy installation artifacts to be usable in host running Ansible
-    cp -rf $kud_inventory_folder/artifacts \
-        /opt/kud/multi-cluster/$cluster_name/
+    install_host_artifacts $1
 
     _print_kubernetes_info
 }