#!/bin/bash # # ============LICENSE_START==================================================== # Copyright (C) 2022-2024 Nordix Foundation. # ============================================================================= # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END====================================================== # This script spins up kubernetes cluster in Microk8s for deploying policy helm charts. # Runs CSITs in kubernetes. WORKSPACE=$(git rev-parse --show-toplevel) export WORKSPACE export GERRIT_BRANCH=$(awk -F= '$1 == "defaultbranch" { print $2 }' "${WORKSPACE}"/.gitreview) CSIT_SCRIPT="scripts/run-test.sh" ROBOT_DOCKER_IMAGE="policy-csit-robot" POLICY_CLAMP_ROBOT="policy-clamp-test.robot clamp-slas.robot" POLICY_API_ROBOT="api-test.robot api-slas.robot" POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot" POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot apex-slas.robot" POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot" POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot" POLICY_DISTRIBUTION_ROBOT="distribution-test.robot" POLICY_API_CONTAINER="policy-api" POLICY_PAP_CONTAINER="policy-pap" POLICY_CLAMP_CONTAINER="policy-clamp-runtime-acm" POLICY_APEX_CONTAINER="policy-apex-pdp" POLICY_DROOLS_CONTAINER="policy-drools-pdp" POLICY_XACML_CONTAINER="policy-xacml-pdp" POLICY_DISTRIBUTION_CONTAINER="policy-distribution" POLICY_K8S_PPNT_CONTAINER="policy-clamp-ac-k8s-ppnt" POLICY_HTTP_PPNT_CONTAINER="policy-clamp-ac-http-ppnt" POLICY_SIM_PPNT_CONTAINER="policy-clamp-ac-sim-ppnt" POLICY_PF_PPNT_CONTAINER="policy-clamp-ac-pf-ppnt" JAEGER_CONTAINER="jaeger" KAFKA_CONTAINER="kafka-deployment" ZK_CONTAINER="zookeeper-deployment" KAFKA_DIR=${WORKSPACE}/helm/cp-kafka SET_VALUES="" DISTRIBUTION_CSAR=${WORKSPACE}/csit/resources/tests/data/csar DIST_TEMP_FOLDER=/tmp/distribution export PROJECT="" export ROBOT_FILE="" export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives export READINESS_CONTAINERS=() function spin_microk8s_cluster() { echo "Verify if Microk8s cluster is running.." microk8s version exitcode="${?}" if [ "$exitcode" -ne 0 ]; then echo "Microk8s cluster not available, Spinning up the cluster.." sudo snap install microk8s --classic --channel=1.30/stable if [ "${?}" -ne 0 ]; then echo "Failed to install kubernetes cluster. Aborting.." return 1 fi echo "Microk8s cluster installed successfully" sudo usermod -a -G microk8s $USER echo "Enabling DNS and Storage plugins" sudo microk8s.enable dns hostpath-storage echo "Creating configuration file for Microk8s" sudo mkdir -p $HOME/.kube sudo chown -R $USER:$USER $HOME/.kube sudo microk8s kubectl config view --raw >$HOME/.kube/config sudo chmod 600 $HOME/.kube/config echo "K8s installation completed" echo "----------------------------------------" else echo "K8s cluster is already running" echo "----------------------------------------" fi echo "Verify if kubectl is running.." kubectl version exitcode="${?}" if [ "$exitcode" -ne 0 ]; then echo "Kubectl not available, Installing.." sudo snap install kubectl --classic --channel=1.30/stable if [ "${?}" -ne 0 ]; then echo "Failed to install Kubectl. Aborting.." return 1 fi echo "Kubectl installation completed" echo "----------------------------------------" else echo "Kubectl is already running" echo "----------------------------------------" return 0 fi echo "Verify if helm is running.." helm version exitcode="${?}" if [ "$exitcode" -ne 0 ]; then echo "Helm not available, Installing.." sudo snap install helm --classic --channel=3.7 if [ "${?}" -ne 0 ]; then echo "Failed to install Helm client. Aborting.." return 1 fi echo "Helm installation completed" echo "----------------------------------------" else echo "Helm is already running" echo "----------------------------------------" return 0 fi } function install_kafka() { echo "Installing Confluent kafka" kubectl apply -f $KAFKA_DIR/zookeeper.yaml kubectl apply -f $KAFKA_DIR/kafka.yaml echo "----------------------------------------" } function uninstall_policy() { echo "Removing the policy helm deployment" sudo helm uninstall csit-policy sudo helm uninstall prometheus sudo helm uninstall csit-robot sudo kubectl delete deploy $ZK_CONTAINER $KAFKA_CONTAINER rm -rf ${WORKSPACE}/helm/policy/Chart.lock if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then sudo helm uninstall policy-chartmuseum sudo helm repo remove chartmuseum-git policy-chartmuseum fi sudo rm -rf /dockerdata-nfs/mariadb-galera/ sudo kubectl delete pvc --all echo "Policy deployment deleted" echo "Clean up docker" docker image prune -f } function teardown_cluster() { echo "Removing k8s cluster and k8s configuration file" sudo snap remove microk8s;rm -rf $HOME/.kube/config sudo snap remove helm; sudo snap remove kubectl; echo "MicroK8s Cluster removed" } function build_robot_image() { echo "Build docker image for robot framework" cd ${WORKSPACE}/csit/resources || exit clone_models if [ "${PROJECT}" == "distribution" ] || [ "${PROJECT}" == "policy-distribution" ]; then copy_csar_file fi echo "Build robot framework docker image" docker login -u docker -p docker nexus3.onap.org:10001 docker build . --file Dockerfile \ --build-arg CSIT_SCRIPT="$CSIT_SCRIPT" \ --build-arg ROBOT_FILE="$ROBOT_FILE" \ --tag "${ROBOT_DOCKER_IMAGE}" --no-cache echo "---------------------------------------------" } function start_csit() { build_robot_image if [ "${?}" -eq 0 ]; then echo "Importing robot image into microk8s registry" docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest sudo microk8s ctr image import policy-csit-robot.tar rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar rm -rf ${WORKSPACE}/csit/resources/tests/models/ echo "---------------------------------------------" if [ "$PROJECT" == "clamp" ] || [ "$PROJECT" == "policy-clamp" ]; then POD_READY_STATUS="0/1" while [[ ${POD_READY_STATUS} != "1/1" ]]; do echo "Waiting for chartmuseum pod to come up..." sleep 5 POD_READY_STATUS=$(sudo kubectl get pods | grep -e "policy-chartmuseum" | awk '{print $2}') done push_acelement_chart fi echo "Installing Robot framework pod for running CSIT" cd ${WORKSPACE}/helm mkdir -p ${ROBOT_LOG_DIR} sudo helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR print_robot_log fi } function print_robot_log() { count_pods=0 while [[ ${count_pods} -eq 0 ]]; do echo "Waiting for pods to come up..." sleep 5 count_pods=$(sudo kubectl get pods --output name | wc -l) done robotpod=$(sudo kubectl get po | grep policy-csit) podName=$(echo "$robotpod" | awk '{print $1}') echo "The robot tests will begin once the policy components {${READINESS_CONTAINERS[*]}} are up and running..." sudo kubectl wait --for=jsonpath='{.status.phase}'=Running --timeout=18m pod/"$podName" echo "Policy deployment status:" sudo kubectl get po sudo kubectl get all -A echo "Robot Test logs:" sudo kubectl logs -f "$podName" } function clone_models() { # download models examples git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-models.git "${WORKSPACE}"/csit/resources/tests/models # create a couple of variations of the policy definitions sed -e 's!Measurement_vGMUX!ADifferentValue!' \ tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \ >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v1_2.json sed -e 's!"version": "1.0.0"!"version": "2.0.0"!' \ -e 's!"policy-version": 1!"policy-version": 2!' \ tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.json \ >tests/models/models-examples/src/main/resources/policies/vCPE.policy.monitoring.input.tosca.v2.json } function copy_csar_file() { zip -F ${DISTRIBUTION_CSAR}/sample_csar_with_apex_policy.csar \ --out ${DISTRIBUTION_CSAR}/csar_temp.csar -q # Remake temp directory sudo rm -rf "${DIST_TEMP_FOLDER}" sudo mkdir "${DIST_TEMP_FOLDER}" sudo cp ${DISTRIBUTION_CSAR}/csar_temp.csar ${DISTRIBUTION_CSAR}/temp.csar sudo mv ${DISTRIBUTION_CSAR}/temp.csar ${DIST_TEMP_FOLDER}/sample_csar_with_apex_policy.csar } function set_project_config() { echo "Setting project configuration for: $PROJECT" case $PROJECT in clamp | policy-clamp) export ROBOT_FILE=$POLICY_CLAMP_ROBOT export READINESS_CONTAINERS=($POLICY_CLAMP_CONTAINER,$POLICY_APEX_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER, $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER,$JAEGER_CONTAINER) export SET_VALUES="--set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_PF_PPNT_CONTAINER.enabled=true --set $POLICY_K8S_PPNT_CONTAINER.enabled=true --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true --set $POLICY_SIM_PPNT_CONTAINER.enabled=true --set $JAEGER_CONTAINER.enabled=true" install_chartmuseum ;; api | policy-api) export ROBOT_FILE=$POLICY_API_ROBOT export READINESS_CONTAINERS=($POLICY_API_CONTAINER) ;; pap | policy-pap) export ROBOT_FILE=$POLICY_PAP_ROBOT export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_API_CONTAINER,$POLICY_XACML_CONTAINER) export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true" ;; apex-pdp | policy-apex-pdp) export ROBOT_FILE=$POLICY_APEX_PDP_ROBOT export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER) export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true" ;; xacml-pdp | policy-xacml-pdp) export ROBOT_FILE=($POLICY_XACML_PDP_ROBOT) export READINESS_CONTAINERS=($POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_XACML_CONTAINER) export SET_VALUES="--set $POLICY_XACML_CONTAINER.enabled=true" ;; drools-pdp | policy-drools-pdp) export ROBOT_FILE=($POLICY_DROOLS_PDP_ROBOT) export READINESS_CONTAINERS=($POLICY_DROOLS_CONTAINER) export SET_VALUES="--set $POLICY_DROOLS_CONTAINER.enabled=true" ;; distribution | policy-distribution) export ROBOT_FILE=($POLICY_DISTRIBUTION_ROBOT) export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER,$POLICY_DISTRIBUTION_CONTAINER) export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true" ;; *) echo "Unknown project supplied. Enabling all policy charts for the deployment" export READINESS_CONTAINERS=($POLICY_APEX_CONTAINER,$POLICY_API_CONTAINER,$POLICY_PAP_CONTAINER, $POLICY_DISTRIBUTION_CONTAINER,$POLICY_DROOLS_CONTAINER,$POLICY_XACML_CONTAINER, $POLICY_CLAMP_CONTAINER,$POLICY_PF_PPNT_CONTAINER,$POLICY_K8S_PPNT_CONTAINER, $POLICY_HTTP_PPNT_CONTAINER,$POLICY_SIM_PPNT_CONTAINER) export SET_VALUES="--set $POLICY_APEX_CONTAINER.enabled=true --set $POLICY_XACML_CONTAINER.enabled=true --set $POLICY_DISTRIBUTION_CONTAINER.enabled=true --set $POLICY_DROOLS_CONTAINER.enabled=true --set $POLICY_CLAMP_CONTAINER.enabled=true --set $POLICY_PF_PPNT_CONTAINER.enabled=true --set $POLICY_K8S_PPNT_CONTAINER.enabled=true --set $POLICY_HTTP_PPNT_CONTAINER.enabled=true --set $POLICY_SIM_PPNT_CONTAINER.enabled=true" ;; esac } function install_chartmuseum () { echo "---------------------------------------------" echo "Installing Chartmuseum helm repository..." sudo helm repo add chartmuseum-git https://chartmuseum.github.io/charts sudo helm repo update sudo helm install policy-chartmuseum chartmuseum-git/chartmuseum --set env.open.DISABLE_API=false --set service.type=NodePort --set service.nodePort=30208 sudo helm plugin install https://github.com/chartmuseum/helm-push echo "---------------------------------------------" } function push_acelement_chart() { echo "Pushing acelement chart to the chartmuseum repo..." sudo helm repo add policy-chartmuseum http://localhost:30208 # download clamp repo git clone -b "${GERRIT_BRANCH}" --single-branch https://github.com/onap/policy-clamp.git "${WORKSPACE}"/csit/resources/tests/clamp ACELEMENT_CHART=${WORKSPACE}/csit/resources/tests/clamp/examples/src/main/resources/clamp/acm/acelement-helm/acelement sudo helm cm-push $ACELEMENT_CHART policy-chartmuseum sudo helm repo update rm -rf ${WORKSPACE}/csit/resources/tests/clamp/ echo "-------------------------------------------" } function get_pod_name() { pods=$(kubectl get pods --no-headers -o custom-columns=':metadata.name' | grep $1) read -rd '' -a pod_array <<< "$pods" echo "${pod_array[@]}" } wait_for_pods_running() { local namespace="$1" shift local timeout_seconds="$1" shift IFS=',' read -ra pod_names <<< "$@" shift local pending_pods=("${pod_names[@]}") local start_time start_time=$(date +%s) while [ ${#pending_pods[@]} -gt 0 ]; do local current_time current_time=$(date +%s) local elapsed_time elapsed_time=$((current_time - start_time)) if [ "$elapsed_time" -ge "$timeout_seconds" ]; then echo "Timed out waiting for the pods to reach 'Running' state." echo "Printing the current status of the deployment before exiting.." kubectl get po; kubectl describe pods; echo "------------------------------------------------------------" for pod in "${pending_pods[@]}"; do echo "Logs of the pod $pod" kubectl logs $pod echo "---------------------------------------------------------" done exit 1 fi local newly_running_pods=() for pod_name_prefix in "${pending_pods[@]}"; do local pod_names=$(get_pod_name "$pod_name_prefix") IFS=' ' read -r -a pod_array <<< "$pod_names" if [ "${#pod_array[@]}" -eq 0 ]; then echo "*** Error: No pods found for the deployment $pod_name_prefix . Exiting ***" return -1 fi for pod in "${pod_array[@]}"; do local pod_status local pod_ready pod_status=$(kubectl get pod "$pod" -n "$namespace" --no-headers -o custom-columns=STATUS:.status.phase 2>/dev/null) pod_ready=$(kubectl get pod "$pod" -o jsonpath='{.status.containerStatuses[*].ready}') if [ "$pod_status" == "Running" ] && [ "$pod_ready" == "true" ]; then echo "Pod '$pod' in namespace '$namespace' is now in 'Running' state and 'Readiness' is true" else newly_running_pods+=("$pod") echo "Waiting for pod '$pod' in namespace '$namespace' to reach 'Running' and 'Ready' state..." fi done done pending_pods=("${newly_running_pods[@]}") sleep 5 done echo "All specified pods are in the 'Running and Ready' state. Exiting the function." } OPERATION="$1" PROJECT="$2" if [ -z "$3" ] then LOCALIMAGE="false" else LOCALIMAGE="$3" fi if [ $OPERATION == "install" ]; then spin_microk8s_cluster if [ "${?}" -eq 0 ]; then export KAFKA_CONTAINERS=($KAFKA_CONTAINER,$ZK_CONTAINER) install_kafka wait_for_pods_running default 300 $KAFKA_CONTAINERS set_project_config echo "Installing policy helm charts in the default namespace" source ${WORKSPACE}/compose/get-k8s-versions.sh if [ $LOCALIMAGE == "true" ]; then echo "loading local image" source ${WORKSPACE}/compose/get-versions.sh ${WORKSPACE}/compose/loaddockerimage.sh fi cd ${WORKSPACE}/helm || exit sudo helm dependency build policy sudo helm install csit-policy policy ${SET_VALUES} sudo helm install prometheus prometheus wait_for_pods_running default 900 ${READINESS_CONTAINERS[@]} echo "Policy chart installation completed" echo "-------------------------------------------" fi if [ "$PROJECT" ]; then export ROBOT_LOG_DIR=${WORKSPACE}/csit/archives/${PROJECT} echo "CSIT will be invoked from $ROBOT_FILE" echo "Readiness containers: ${READINESS_CONTAINERS[*]}" echo "-------------------------------------------" start_csit else echo "No project supplied for running CSIT" fi elif [ $OPERATION == "uninstall" ]; then uninstall_policy elif [ $OPERATION == "clean" ]; then teardown_cluster else echo "Invalid arguments provided. Usage: $0 [options..] {install {project_name} | uninstall | clean} {uselocalimage = true/false}" fi