diff options
| -rw-r--r-- | kud/deployment_infra/playbooks/configure-topology-manager.yml | 66 | ||||
| -rw-r--r-- | kud/deployment_infra/playbooks/kud-vars.yml | 7 | ||||
| -rwxr-xr-x | kud/hosting_providers/vagrant/installer.sh | 4 | ||||
| -rwxr-xr-x | kud/tests/topology-manager.sh | 112 | ||||
| -rwxr-xr-x | kud/tests/vnfs/comp-app/collection/app2/helm/prometheus-operator/values.yaml | 4 | ||||
| -rwxr-xr-x | src/dcm/test/dcm_call_api.sh | 61 |
6 files changed, 208 insertions, 46 deletions
diff --git a/kud/deployment_infra/playbooks/configure-topology-manager.yml b/kud/deployment_infra/playbooks/configure-topology-manager.yml new file mode 100644 index 00000000..012bc8b0 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-topology-manager.yml @@ -0,0 +1,66 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-node + tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + + - name: creating kubelet config + become: yes + blockinfile: + path: "{{ kubernetes_config_file }}" + marker: "# {mark} OpenNESS configuration - General" + create: yes + block: | + featureGates: + TopologyManager: {{ False if topology_manager.policy == 'none' else True }} + notify: + - enable and restart kubelet + + - name: customize kubelet config - CPU Manager + become: yes + blockinfile: + path: "{{ kubernetes_config_file }}" + marker: "# {mark} OpenNESS configuration - CPU Manager" + block: | + cpuManagerPolicy: {{ cpu_manager.policy }} + state: "{{ 'present' if cpu_manager.policy == 'static' else 'absent' }}" + notify: + - remove cpu manager checkpoint file + - enable and restart kubelet + + - name: customize kubelet config - Topology Manager + become: yes + blockinfile: + path: "{{ kubernetes_config_file }}" + marker: "# {mark} OpenNESS configuration - Topology Manager" + block: | + topologyManagerPolicy: {{ topology_manager.policy }} + state: "{{ 'absent' if topology_manager.policy == 'none' else 'present' }}" + notify: + - enable and restart kubelet + + handlers: + - name: enable and restart kubelet + become: yes + systemd: + name: kubelet + daemon_reload: yes + enabled: yes + masked: no + state: restarted + + - name: remove cpu manager checkpoint file + become: yes + file: + path: "{{ cpu_manager.checkpoint_file }}" + state: absent diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index 4988a473..30e54f03 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -79,3 +79,10 @@ optane_ipmctl_source_type: "tarball" optane_ipmctl_version: 02.00.00.3474 optane_ipmctl_url: "https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/ipmctl/{{ optane_package }}.tar.xz" optane_ipmctl_package: ipmctl_02.00.00.3474+really01.00.00.3469.orig + +kubernetes_config_file: "/etc/kubernetes/kubelet-config.yaml" +cpu_manager: + policy: "static" # Options: none (disabled), static (default) + checkpoint_file: "/var/lib/kubelet/cpu_manager_state" +topology_manager: + policy: "best-effort" # Options: none (disabled), best-effort (default), restricted, single-numa-node diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 27ab7fc1..71e4d8b7 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -155,13 +155,13 @@ function install_addons { _install_ansible sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov qat optane cmk}; do + for addon in ${KUD_ADDONS:-topology-manager virtlet ovn4nfv nfd sriov qat optane cmk}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log done echo "Run the test cases if testing_enabled is set to true." if [[ "${testing_enabled}" == "true" ]]; then - for addon in ${KUD_ADDONS:-multus virtlet ovn4nfv nfd sriov qat optane cmk}; do + for addon in ${KUD_ADDONS:-multus topology-manager virtlet ovn4nfv nfd sriov qat optane cmk}; do pushd $kud_tests bash ${addon}.sh popd diff --git a/kud/tests/topology-manager.sh b/kud/tests/topology-manager.sh new file mode 100755 index 00000000..bbffd4d7 --- /dev/null +++ b/kud/tests/topology-manager.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +source _common.sh +source _functions.sh + +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) +if [ -z "$ethernet_adpator_version" ]; then + echo " Ethernet adapator version is not set. Topology manager test case cannot run on this machine" + exit 0 +else + echo "NIC card specs match. Topology manager option avaiable for this version." +fi + +pod_name=pod-topology-manager +csar_id=bd55cccc-bf34-11ea-b3de-0242ac130004 + +function create_pod_yaml { + local csar_id=$1 + _checks_args $csar_id + pushd ${CSAR_DIR}/${csar_id} + + cat << POD > $pod_name.yaml +kind: Pod +apiVersion: v1 +metadata: + name: $pod_name + annotations: + k8s.v1.cni.cncf.io/networks: sriov-eno2 +spec: + containers: + - name: $pod_name + image: docker.io/centos/tools:latest + command: + - /sbin/init + resources: + limits: + cpu: "1" + memory: "500Mi" + intel.com/intel_sriov_700: '1' + requests: + cpu: "1" + memory: "500Mi" + intel.com/intel_sriov_700: '1' +POD + popd +} + +create_pod_yaml ${csar_id} +kubectl delete pod $pod_name --ignore-not-found=true --now --wait +kubectl create -f ${CSAR_DIR}/${csar_id}/$pod_name.yaml --validate=false + +status_phase="" +while [[ $status_phase != "Running" ]]; do + new_phase=$(kubectl get pods $pod_name | awk 'NR==2{print $3}') + if [[ $new_phase != $status_phase ]]; then + echo "$(date +%H:%M:%S) - $pod_name : $new_phase" + status_phase=$new_phase + fi + if [[ $new_phase == "Running" ]]; then + echo "Pod is up and running.." + fi + if [[ $new_phase == "Err"* ]]; then + exit 1 + fi +done + +container_id=$(kubectl describe pod $pod_name | grep "Container ID" | awk '{print $3}' ) +container_id=${container_id#docker://} +container_id=${container_id:0:12} + +apt-get install -y jq +cpu_core=$(cat /var/lib/kubelet/cpu_manager_state | jq -r .| grep ${container_id} | awk -F ':' '{print $2}'| awk -F '"' '{print $2}') +numa_node_number=$(lscpu | grep "NUMA node(s)" | awk -F ':' '{print $2}') +for (( node=0; node<$numa_node_number; node++ )); do + ranges=$(lscpu | grep "NUMA node"$node | awk -F ':' '{print $2}') + ranges=(${ranges//,/ }) + for range in ${ranges[@]}; do + min=$(echo $range | awk -F '-' '{print $1}') + max=$(echo $range | awk -F '-' '{print $2}') + if [ $cpu_core -ge $min ] && [ $cpu_core -le $max ]; then + cpu_numa_node=$node + fi + done +done + +vf_pci=$(kubectl exec -it $pod_name env | grep PCIDEVICE_INTEL_COM_INTEL_SRIOV_700 | awk -F '=' '{print $2}' | sed 's/\r//g') +vf_numa_node=$(cat /sys/bus/pci/devices/$vf_pci/numa_node) + +echo "The allocated cpu core is:" $cpu_core +echo "The numa node of the allocated cpu core is:" $cpu_numa_node +echo "The PCI address of the allocated vf is:" $vf_pci +echo "The numa node of the allocated vf is:" $vf_numa_node +if [ $cpu_numa_node == $vf_numa_node ]; then + echo "The allocated cpu core and vf are on the same numa node" +else + echo "The allocated cpu core and vf are on different numa nodes" +fi + +kubectl delete pod $pod_name --now +echo "Test complete." diff --git a/kud/tests/vnfs/comp-app/collection/app2/helm/prometheus-operator/values.yaml b/kud/tests/vnfs/comp-app/collection/app2/helm/prometheus-operator/values.yaml index f4332e7f..f296ca50 100755 --- a/kud/tests/vnfs/comp-app/collection/app2/helm/prometheus-operator/values.yaml +++ b/kud/tests/vnfs/comp-app/collection/app2/helm/prometheus-operator/values.yaml @@ -1162,11 +1162,11 @@ prometheusOperator: ## Deploy CRDs used by Prometheus Operator. ## - createCustomResource: false + createCustomResource: true ## Attempt to clean up CRDs created by Prometheus Operator. ## - cleanupCustomResource: false + cleanupCustomResource: true ## Labels to add to the operator pod ## diff --git a/src/dcm/test/dcm_call_api.sh b/src/dcm/test/dcm_call_api.sh index 966bc3d6..33fbf314 100755 --- a/src/dcm/test/dcm_call_api.sh +++ b/src/dcm/test/dcm_call_api.sh @@ -14,7 +14,6 @@ # * limitations under the License. # */ - dcm_addr="http://localhost:9077" # parameters @@ -22,6 +21,7 @@ project="test-project" description="test-description" logical_cloud_name="lc1" namespace="ns1" +api_groups="" user="user-1" permission="permission-1" cluster_provider_name="cp-1" @@ -52,7 +52,7 @@ logical_cloud_data="$(cat << EOF "type" : "certificate", "user-permissions" : [ { "permission-name" : "${permission}", - "apiGroups" : ["stable.example.com"], + "apiGroups" : ["${api_groups}"], "resources" : ["secrets", "pods"], "verbs" : ["get", "watch", "list", "create"] } @@ -99,8 +99,6 @@ cluster_2_data="$(cat << EOF EOF )" -# removed all special chars from quota spec keys -# due to loss of data when unmarshalling from json quota_data="$(cat << EOF { "metadata" : { @@ -108,54 +106,34 @@ quota_data="$(cat << EOF "description": "${description}" }, "spec" : { - "persistentvolumeclaims" : "10", - "pods": "500", - "configmaps" : "10", - "replicationcontrollers": "10", - "resourcequotas" : "10", - "services": "10", - "secrets" : "10" - } -} -EOF -)" - -quota_data_original="$(cat << EOF -{ - "metadata" : { - "name" : "${quota_name}", - "description": "${description}" - }, - "spec" : { "limits.cpu": "400", "limits.memory": "1000Gi", "requests.cpu": "300", "requests.memory": "900Gi", "requests.storage" : "500Gi", - "requests.ephemeral-storage": "", - "limits.ephemeral-storage": "", - "persistentvolumeclaims" : " ", + "requests.ephemeral-storage": "500", + "limits.ephemeral-storage": "500", + "persistentvolumeclaims" : "500", "pods": "500", - "configmaps" : "", - "replicationcontrollers": "", - "resourcequotas" : "", - "services": "", - "services.loadbalancers" : "", - "services.nodeports" : "", - "secrets" : "", - "count/replicationcontrollers" : "", - "count/deployments.apps" : "", - "count/replicasets.apps" : "", - "count/statefulsets.apps" : "", - "count/jobs.batch" : "", - "count/cronjobs.batch" : "", - "count/deployments.extensions" : "" + "configmaps" : "1000", + "replicationcontrollers": "500", + "resourcequotas" : "500", + "services": "500", + "services.loadbalancers" : "500", + "services.nodeports" : "500", + "secrets" : "500", + "count/replicationcontrollers" : "500", + "count/deployments.apps" : "500", + "count/replicasets.apps" : "500", + "count/statefulsets.apps" : "500", + "count/jobs.batch" : "500", + "count/cronjobs.batch" : "500", + "count/deployments.extensions" : "500" } } EOF )" - # Create logical cloud printf "\n\nCreating logical cloud data\n\n" curl -d "${logical_cloud_data}" -X POST ${logical_cloud_url} @@ -169,7 +147,6 @@ curl -d "${cluster_2_data}" -X POST ${cluster_url} printf "\n\nAdding resource quota for the logical cloud\n\n" curl -d "${quota_data}" -X POST ${quota_url} - # Get logical cloud data printf "\n\nGetting logical cloud\n\n" curl -X GET "${logical_cloud_url}/${logical_cloud_name}" |