diff options
Diffstat (limited to 'kud')
39 files changed, 1727 insertions, 17 deletions
diff --git a/kud/build/Dockerfile b/kud/build/Dockerfile new file mode 100644 index 00000000..38c63295 --- /dev/null +++ b/kud/build/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:18.04 as base +ARG KUD_ENABLE_TESTS=false +ARG KUD_PLUGIN_ENABLED=false +ENV KUD_ENABLE_TESTS=$KUD_ENABLE_TESTS +ENV KUD_PLUGIN_ENABLED=$KUD_PLUGIN_ENABLED +ADD . /usr/src/multicloud-k8s +USER root +SHELL ["/bin/bash", "-c"] +WORKDIR /usr/src/multicloud-k8s/kud/hosting_providers/containerized +RUN ./installer --install_pkg +ENTRYPOINT ["tail -f /dev/null"] diff --git a/kud/deployment_infra/galaxy-requirements.yml b/kud/deployment_infra/galaxy-requirements.yml index 17ac1dc2..3191dc19 100644 --- a/kud/deployment_infra/galaxy-requirements.yml +++ b/kud/deployment_infra/galaxy-requirements.yml @@ -10,6 +10,6 @@ - src: andrewrothstein.go version: v2.1.15 - src: andrewrothstein.kubernetes-helm - version: v1.2.9 + version: v1.2.17 - src: geerlingguy.docker version: 2.5.2 diff --git a/kud/deployment_infra/images/multus-daemonset.yml b/kud/deployment_infra/images/multus-daemonset.yml index ff44a217..0c41a052 100644 --- a/kud/deployment_infra/images/multus-daemonset.yml +++ b/kud/deployment_infra/images/multus-daemonset.yml @@ -79,7 +79,7 @@ data: "delegates": [ { "cniVersion": "0.3.1", - "name": "default-cni-network", + "name": "cni0", "plugins": [ { "type": "flannel", diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml new file mode 100644 index 00000000..bd943d04 --- /dev/null +++ b/kud/deployment_infra/images/sriov-cni.yml @@ -0,0 +1,45 @@ +# SRIOV-CNI Release v1 +# Based on: +# https://github.com/intel/sriov-cni/blob/master/images/sriov-cni-daemonset.yaml +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-cni-ds-amd64 + namespace: kube-system + labels: + tier: node + app: sriov-cni +spec: + template: + metadata: + labels: + tier: node + app: sriov-cni + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: kube-sriov-cni + image: nfvpe/sriov-cni + securityContext: + privileged: true + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml new file mode 100644 index 00000000..72f33869 --- /dev/null +++ b/kud/deployment_infra/images/sriov-daemonset.yml @@ -0,0 +1,82 @@ +# SRIOV device CNI plugin +# Based on: +# https://github.com/intel/sriov-network-device-plugin/blob/master/images/sriovdp-daemonset.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sriovdp-config + namespace: kube-system +data: + config.json: | + { + "resourceList": [{ + "resourceName": "intel_sriov_700", + "selectors": { + "vendors": ["8086"], + "drivers": ["i40evf", "iavf"] + } + }] + } + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-device-plugin + namespace: kube-system + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-sriov-device-plugin-amd64 + namespace: kube-system + labels: + tier: node + app: sriovdp +spec: + template: + metadata: + labels: + tier: node + app: sriovdp + spec: + hostNetwork: true + hostPID: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: sriov-device-plugin + containers: + - name: kube-sriovdp + image: nfvpe/sriov-device-plugin + args: + - --log-dir=sriovdp + - --log-level=10 + securityContext: + privileged: true + volumeMounts: + - name: devicesock + mountPath: /var/lib/kubelet/ + readOnly: false + - name: log + mountPath: /var/log + - name: config-volume + mountPath: /etc/pcidp + volumes: + - name: devicesock + hostPath: + path: /var/lib/kubelet/ + - name: log + hostPath: + path: /var/log + - name: config-volume + configMap: + name: sriovdp-config + items: + - key: config.json + path: config.json diff --git a/kud/deployment_infra/playbooks/Debian.yml b/kud/deployment_infra/playbooks/Debian.yml index 96357fe2..b9725b2d 100644 --- a/kud/deployment_infra/playbooks/Debian.yml +++ b/kud/deployment_infra/playbooks/Debian.yml @@ -11,8 +11,6 @@ openvswitch_service: openvswitch-switch openvswitch_pkgs: - openvswitch-common - openvswitch-switch - - libopenvswitch - - openvswitch-datapath-dkms ovn_central_service: ovn-central ovn_central_pkgs: - ovn-central # <= 2.8.1-1 diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml new file mode 100644 index 00000000..6adaf2ee --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml @@ -0,0 +1,56 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-master + tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + + - name: Change the onap4k8s directory and run helm delete + command: /usr/local/bin/helm delete --purge multicloud-onap8ks + register: helm_delete + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: helm_delete.stdout_lines + + - name: Change the onap4k8s directory and delete the ona4k8s-ns namespace + command: /usr/local/bin/kubectl delete ns onap4k8s-ns + register: delete_onap_ns + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: delete_onap_ns.stdout_lines + + - name: Change the onap4k8s directory and make clean + command: /usr/bin/make clean + register: make_clean + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_clean.stdout_lines + + - name: Change the onap4k8s directory and make repo-stop + command: /usr/bin/make repo-stop + register: make_repo_stop + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_repo_stop.stdout_lines + + - name: clean multicloud-k8s path + file: + state: absent + path: /opt/multicloud diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s.yml b/kud/deployment_infra/playbooks/configure-onap4k8s.yml new file mode 100644 index 00000000..11729171 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-onap4k8s.yml @@ -0,0 +1,55 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-master + tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + + - name: Getting onap4k8s code in /opt folder + git: + repo: 'https://github.com/onap/multicloud-k8s.git' + dest: /opt/multicloud + + - name: install make package for ubuntu systems + apt: name=make state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install make package for centos systems + yum: name=make state=present update_cache=yes + when: ansible_distribution == "CentOS" + + - name: Change the onap4k8s directory and run the command make repo + command: /usr/bin/make repo + register: make_repo + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_repo.stdout_lines + + - name: Change the onap4k8s directory and run the command make all + command: /usr/bin/make all + register: make_all + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: make_all.stdout_lines + + - name: Change the onap4k8s directory and run the command helm install + command: /usr/local/bin/helm install dist/packages/multicloud-k8s-5.0.0.tgz --name multicloud-onap8ks --namespace onap4k8s-ns --set service.type=NodePort + register: helm_install + args: + chdir: /opt/multicloud/deployments/helm/onap4k8s + + - debug: + var: helm_install.stdout_lines diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml new file mode 100644 index 00000000..45f276c6 --- /dev/null +++ b/kud/deployment_infra/playbooks/configure-sriov.yml @@ -0,0 +1,29 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- import_playbook: preconfigure-sriov.yml + +- hosts: localhost + become: yes + tasks: + - debug: + var: SRIOV_NODE + - name: Apply Multus + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml" + when: SRIOV_NODE + - name: Apply SRIOV CNI + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml" + when: SRIOV_NODE + - name: Apply SRIOV DaemonSet + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml" + when: SRIOV_NODE + - name: Apply SRIOV Network Attachment definition + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml" + when: SRIOV_NODE diff --git a/kud/deployment_infra/playbooks/install_iavf_drivers.sh b/kud/deployment_infra/playbooks/install_iavf_drivers.sh new file mode 100755 index 00000000..7a54e9f2 --- /dev/null +++ b/kud/deployment_infra/playbooks/install_iavf_drivers.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Based on: +# https://gerrit.akraino.org/r/#/c/icn/+/1359/1/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh + +nic_models=(XL710 X722) +nic_drivers=(i40e) +device_checkers=(is_not_used is_driver_match is_model_match) + +function install_iavf_driver { + local ifname=$1 + + echo "Installing modules..." + echo "Installing i40evf blacklist file..." + mkdir -p "/etc/modprobe.d/" + echo "blacklist i40evf" > "/etc/modprobe.d/iavf-blacklist-i40evf.conf" + + kver=`uname -a | awk '{print $3}'` + install_mod_dir=/lib/modules/$kver/updates/drivers/net/ethernet/intel/iavf/ + echo "Installing driver in $install_mod_dir" + mkdir -p $install_mod_dir + cp iavf.ko $install_mod_dir + + echo "Installing kernel module i40evf..." + depmod -a + modprobe i40evf + modprobe iavf + + echo "Enabling VF on interface $ifname..." + echo "/sys/class/net/$ifname/device/sriov_numvfs" + echo '8' > /sys/class/net/$ifname/device/sriov_numvfs +} + +function is_not_used { + local ifname=$1 + route_info=`ip route show | grep $ifname` + if [ -z "$route_info" ]; then + return 1 + else + return 0 + fi +} + +function is_driver_match { + local ifname=$1 + driver=`cat /sys/class/net/$ifname/device/uevent | grep DRIVER | cut -f2 -d "="` + if [ ! -z "$driver" ]; then + for nic_driver in ${nic_drivers[@]}; do + if [ "$driver" = "$nic_driver" ]; then + return 1 + fi + done + fi + return 0 +} + +function is_model_match { + local ifname=$1 + pci_addr=`cat /sys/class/net/$ifname/device/uevent | grep PCI_SLOT_NAME | cut -f2 -d "=" | cut -f2,3 -d ":"` + if [ ! -z "$pci_addr" ]; then + for nic_model in ${nic_models[@]}; do + model_match=$(lspci | grep $pci_addr | grep $nic_model) + if [ ! -z "$model_match" ]; then + return 1 + fi + done + fi + return 0 +} + +function get_sriov_ifname { + for net_device in /sys/class/net/*/ ; do + if [ -e $net_device/device/sriov_numvfs ] ; then + ifname=$(basename $net_device) + for device_checker in ${device_checkers[@]}; do + eval $device_checker $ifname + if [ "$?" = "0" ]; then + ifname="" + break + fi + done + if [ ! -z "$ifname" ]; then + echo $ifname + return + fi + fi + done + echo '' +} + +if [ $# -ne 1 ] ; then + ifname=$(get_sriov_ifname) + if [ -z "$ifname" ]; then + echo "Cannot find Nic with SRIOV support." + else + install_iavf_driver $ifname + fi +else + ifname=$1 + if [ ! -e /sys/class/net/$ifname/device/sriov_numvfs ] ; then + echo "${ifname} is not a valid sriov interface" + else + install_iavf_driver $ifname + fi +fi diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml index a9910f8d..2a25049a 100644 --- a/kud/deployment_infra/playbooks/kud-vars.yml +++ b/kud/deployment_infra/playbooks/kud-vars.yml @@ -39,8 +39,14 @@ istio_source_type: "tarball" istio_version: 1.0.3 istio_url: "https://github.com/istio/istio/releases/download/{{ istio_version }}/istio-{{ istio_version }}-linux.tar.gz" +sriov_dest: "{{ base_dest }}/sriov" +driver_source_type: "tarball" +driver_version: 3.7.34 +driver_url: "https://downloadmirror.intel.com/28943/eng/iavf-{{ driver_version }}.tar.gz" +package: iavf-3.7.34 + go_version: '1.12.5' kubespray_version: 2.10.4 -helm_client_version: 2.9.1 +helm_client_version: 2.13.1 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634 ansible_version: 2.7.10 diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml new file mode 100644 index 00000000..fd16d935 --- /dev/null +++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml @@ -0,0 +1,118 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: kube-node + become: yes + pre_tasks: + - name: Create SRIOV driver folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - sriov + - copy: + src: "{{ playbook_dir }}/sriov_hardware_check.sh" + dest: sriov + - name: Changing perm of "sh", adding "+x" + shell: "chmod +x sriov_hardware_check.sh" + args: + chdir: "sriov" + warn: False + - name: Register SRIOV + shell: "echo {{ SRIOV | default(False) }}" + - name: Run the script and Re-evaluate the variable + command: sriov/sriov_hardware_check.sh + register: output + - set_fact: + _SRIOV: "{{ output.stdout }}" + - name: Recreate the conf file for every host + file: + path: /tmp/sriov.conf + state: absent + delegate_to: localhost + - lineinfile : > + dest=/tmp/sriov.conf + create=yes + line='{{_SRIOV}}' + delegate_to: localhost + - name: Clean the script and folder. + file: + path: sriov + state: absent + +# Run the following task only if the SRIOV is set to True +# i.e when SRIOV hardware is available +- hosts: localhost + become: yes + pre_tasks: + - name: Read SRIOV value from the conf file. + command: cat /tmp/sriov.conf + register: installer_output + become: yes + - set_fact: + SRIOV_NODE: "{{ installer_output.stdout }}" + - name: Load kud variables + include_vars: + file: kud-vars.yml + when: SRIOV_NODE + tasks: + - name: Create sriov folder + file: + state: directory + path: "{{ sriov_dest }}" + ignore_errors: yes + when: SRIOV_NODE + - name: Get SRIOV compatible driver + get_url: "url={{ driver_url }} dest=/tmp/{{ package }}.tar.gz" + when: SRIOV_NODE + - name: Extract sriov source code + unarchive: + src: "/tmp/{{ package }}.tar.gz" + dest: "{{ sriov_dest }}" + when: SRIOV_NODE + - name: Build the default target + make: + chdir: "/tmp/sriov/{{ package }}/src" + become: yes + when: SRIOV_NODE +# Copy all the driver and install script into target node +- hosts: kube-node + become: yes + pre_tasks: + - name: Load kud variables + include_vars: + file: kud-vars.yml + when: _SRIOV + tasks: + - name: create SRIOV driver folder in the target destination + file: + state: directory + path: "{{ item }}" + with_items: + - sriov_driver + when: _SRIOV + - copy: + src: "{{ sriov_dest }}/{{ package }}/src/iavf.ko" + dest: sriov_driver + remote_src: no + when: _SRIOV + - copy: + src: "{{ playbook_dir }}/install_iavf_drivers.sh" + dest: sriov_driver/install.sh + remote_src: no + when: _SRIOV + - name: Changing perm of "install.sh", adding "+x" + file: dest=sriov_driver/install.sh mode=a+x + when: _SRIOV + - name: Run a script with arguments + shell: ./install.sh + args: + chdir: "sriov_driver" + when: _SRIOV diff --git a/kud/deployment_infra/playbooks/sriov-nad.yml b/kud/deployment_infra/playbooks/sriov-nad.yml new file mode 100644 index 00000000..7670b700 --- /dev/null +++ b/kud/deployment_infra/playbooks/sriov-nad.yml @@ -0,0 +1,19 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-eno2 + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/intel_sriov_700 +spec: + config: '{ + "type": "sriov", + "cniVersion": "0.3.1", + "ipam": { + "type": "host-local", + "subnet": "10.56.206.0/24", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "10.56.206.1" + } + }' diff --git a/kud/deployment_infra/playbooks/sriov_hardware_check.sh b/kud/deployment_infra/playbooks/sriov_hardware_check.sh new file mode 100644 index 00000000..662c28c8 --- /dev/null +++ b/kud/deployment_infra/playbooks/sriov_hardware_check.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail + +source /etc/environment + +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) +if [ -z "$ethernet_adpator_version" ]; then + echo "False" + exit 0 +fi +SRIOV_ENABLED=${ethernet_adpator_version:-"false"} +#checking for the right hardware version of NIC on the machine +if [ "$ethernet_adpator_version" == "XL710" ]; then + echo "True" +else + echo "False" +fi diff --git a/kud/hosting_providers/containerized/README.md b/kud/hosting_providers/containerized/README.md new file mode 100644 index 00000000..12ce1a19 --- /dev/null +++ b/kud/hosting_providers/containerized/README.md @@ -0,0 +1,141 @@ +# Multi cluster installation + +## Introduction + +Multi Cluster installation is an important features for production deployments. + +Most of the project are using the Kubernetes as undercloud orchestration. So deploying multi cluster for the multi cloud region should be maintained by Kubernetes + +This section explains how to deploy the Multi cluster of Kubernetes from a containerized KUD running as a Kubernetes Job. + +## How it works + +KUD installation installer is divided into two regions with args - `--install-pkg` and `--cluster <cluster-name>` + +### Args +**--install-pkg** - Installs packages required to run installer script itself inside a container and kubespray packages + +**--cluster < cluster-name >** - Installs k8s cluster, addons and plugins and store the artifacts in the host machine + +### Internal Mechanism + +* Container image is build using the `installer --install-pkg` arg and Kubernetes job is used to install the cluster using `installer --cluster <cluster-name>`. Installer will invoke the kubespray cluster.yml, kud-addsons and plugins ansible cluster. + +Installer script finds the `hosts.init` for each cluster in `/opt/multi-cluster/<cluster-name>` + +Kubernetes jobs(a cluster per job) are used to install multiple clusters and logs of each cluster deployments are stored in the `/opt/kud/multi-cluster/<cluster-name>/logs` and artifacts are stored as follows `/opt/kud/multi-cluster/<cluster-name>/artifacts` + +## Quickstart Installation Guide + +Build the kud docker images as follows, add KUD_ENABLE_TESTS & KUD_PLUGIN_ENABLED for the testing only: + +``` +$ git clone https://github.com/onap/multicloud-k8s.git && cd multicloud-k8s +$ docker build --rm \ + --build-arg http_proxy=${http_proxy} \ + --build-arg HTTP_PROXY=${HTTP_PROXY} \ + --build-arg https_proxy=${https_proxy} \ + --build-arg HTTPS_PROXY=${HTTPS_PROXY} \ + --build-arg no_proxy=${no_proxy} \ + --build-arg NO_PROXY=${NO_PROXY} \ + --build-arg KUD_ENABLE_TESTS=true \ + --build-arg KUD_PLUGIN_ENABLED=true \ + -t github.com/onap/multicloud-k8s:latest . -f build/Dockerfile +``` +Let's create a cluster-101 and cluster-102 hosts.ini as follows + +``` +$ mkdir -p /opt/kud/multi-cluster/{cluster-101,cluster-102} +``` + +Create hosts.ini as follows in the direcotry cluster-101(c01 IP address 10.10.10.3) and cluster-102(c02 IP address 10.10.10.5) + +``` +/opt/kud/multi-cluster/cluster-101/hosts.ini +[all] +c01 ansible_ssh_host=10.10.10.5 ansible_ssh_port=22 + +[kube-master] +c01 + +[kube-node] +c01 + +[etcd] +c01 + +[ovn-central] +c01 + +[ovn-controller] +c01 + +[virtlet] +c01 + +[k8s-cluster:children] +kube-node +kube-master +``` +Do the same for the cluster-102 with c01 and IP address 10.10.10.5. + +Create the ssh secret for Baremetal or VM based on your deployment. and Launch the kubernetes job as follows +``` +$ kubectl create secret generic ssh-key-secret --from-file=id_rsa=/root/.ssh/id_rsa --from-file=id_rsa.pub=/root/.ssh/id_rsa.pub +$ CLUSTER_NAME=cluster-101 +$ cat <<EOF | kubectl create -f - +apiVersion: batch/v1 +kind: Job +metadata: + name: kud-$CLUSTER_NAME +spec: + template: + spec: + hostNetwork: true + containers: + - name: kud + image: github.com/onap/multicloud-k8s:latest + imagePullPolicy: IfNotPresent + volumeMounts: + - name: multi-cluster + mountPath: /opt/kud/multi-cluster + - name: secret-volume + mountPath: "/.ssh" + command: ["/bin/sh","-c"] + args: ["cp -r /.ssh /root/; chmod -R 600 /root/.ssh; ./installer --cluster $CLUSTER_NAME --plugins onap4k8s"] + securityContext: + privileged: true + volumes: + - name: multi-cluster + hostPath: + path: /opt/kud/multi-cluster + - name: secret-volume + secret: + secretName: ssh-key-secret + restartPolicy: Never + backoffLimit: 0 + +EOF +``` + +Multi - cluster information from the host machine; + +``` +$ kubectl --kubeconfig=/opt/kud/multi-cluster/cluster-101/artifacts/admin.conf cluster-info +Kubernetes master is running at https://192.168.121.2:6443 +coredns is running at https://192.168.121.2:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy +kubernetes-dashboard is running at https://192.168.121.2:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy + +To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. +$ kubectl --kubeconfig=/opt/kud/multi-cluster/cluster-102/artifacts/admin.conf cluster-info +Kubernetes master is running at https://192.168.121.6:6443 +coredns is running at https://192.168.121.6:6443/api/v1/namespaces/kube-system/services/coredns:dns/proxy +kubernetes-dashboard is running at https://192.168.121.6:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy + +To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. +``` + + +## License + +Apache-2.0 diff --git a/kud/hosting_providers/containerized/installer b/kud/hosting_providers/containerized/installer new file mode 120000 index 00000000..2b6cb163 --- /dev/null +++ b/kud/hosting_providers/containerized/installer @@ -0,0 +1 @@ +installer.sh
\ No newline at end of file diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh new file mode 100755 index 00000000..f1b95acb --- /dev/null +++ b/kud/hosting_providers/containerized/installer.sh @@ -0,0 +1,312 @@ +#!/bin/bash +#SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail +set -ex + +INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +function install_prerequisites { +#install package for docker images + apt-get update + apt-get install -y curl vim wget git \ + software-properties-common python-pip sudo + add-apt-repository -y ppa:longsleep/golang-backports + apt-get update + apt-get install -y golang-go rsync +} + +# _install_ansible() - Install and Configure Ansible program +function _install_ansible { + local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml | + awk -F ': ' '{print $2}') + mkdir -p /etc/ansible/ + pip install ansible==$version +} + +# install_k8s() - Install Kubernetes using kubespray tool +function install_kubespray { + echo "Deploying kubernetes" + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \ + awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" \ + $kud_inventory_folder/group_vars/k8s-cluster.yml | \ + awk -F "\"" '{print $2}') + local tarball=v$version.tar.gz + # install make to run mitogen target & unzip is mitogen playbook dependency + apt-get install -y sshpass make unzip + _install_ansible + wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball + tar -C $dest_folder -xzf $tarball + mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + chown -R root:root $dest_folder/kubespray-$version + mkdir -p ${local_release_dir}/containers + rm $tarball + + pushd $dest_folder/kubespray-$version/ + pip install -r ./requirements.txt + make mitogen + popd + rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null + if [[ -n "${verbose:-}" ]]; then + echo "kube_log_level: 5" | tee \ + $kud_inventory_folder/group_vars/all.yml + else + echo "kube_log_level: 2" | tee \ + $kud_inventory_folder/group_vars/all.yml + fi + echo "kubeadm_enabled: true" | \ + tee --append $kud_inventory_folder/group_vars/all.yml + if [[ -n "${http_proxy:-}" ]]; then + echo "http_proxy: \"$http_proxy\"" | tee --append \ + $kud_inventory_folder/group_vars/all.yml + fi + if [[ -n "${https_proxy:-}" ]]; then + echo "https_proxy: \"$https_proxy\"" | tee --append \ + $kud_inventory_folder/group_vars/all.yml + fi +} + +function install_k8s { + version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \ + awk -F ': ' '{print $2}') + local cluster_name=$1 + ansible-playbook $verbose -i \ + $kud_inventory $dest_folder/kubespray-$version/cluster.yml \ + -e cluster_name=$cluster_name --become --become-user=root | \ + tee $cluster_log/setup-kubernetes.log + + # Configure environment + mkdir -p $HOME/.kube + cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config + # Copy Kubespray kubectl to be usable in host running Ansible. + # Requires kubectl_localhost: true in inventory/group_vars/k8s-cluster.yml + if !(which kubectl); then + cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ + fi + + cp -rf $kud_inventory_folder/artifacts \ + /opt/kud/multi-cluster/$cluster_name/ +} + +# install_addons() - Install Kubenertes AddOns +function install_addons { + if [ ${1:+1} ]; then + local plugins_name="$1" + echo "additional addons plugins $1" + else + local plugins_name="" + echo "no additional addons pluigns" + fi + + source /etc/environment + echo "Installing Kubernetes AddOns" + ansible-galaxy install $verbose -r \ + $kud_infra_folder/galaxy-requirements.yml --ignore-errors + + ansible-playbook $verbose -i \ + $kud_inventory $kud_playbooks/configure-kud.yml | \ + tee $cluster_log/setup-kud.log + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do + echo "Deploying $addon using configure-$addon.yml playbook.." + ansible-playbook $verbose -i \ + $kud_inventory $kud_playbooks/configure-${addon}.yml | \ + tee $cluster_log/setup-${addon}.log + done + + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do + pushd $kud_tests + bash ${addon}.sh + popd + done + fi + echo "Add-ons deployment complete..." +} + +# install_plugin() - Install ONAP Multicloud Kubernetes plugin +function install_plugin { + echo "Installing multicloud/k8s onap4k8s plugin" + if [[ "${testing_enabled}" == "true" ]]; then + pushd $kud_tests + echo "Test the onap4k8s installation" + bash onap4k8s.sh + echo "Test the onap4k8s plugin installation" + for functional_test in plugin_edgex plugin_fw; do + bash ${functional_test}.sh --external + done + popd + fi +} + +# install_controllers() - Install ONAP Multicloud Kubernetes controllers +function install_controllers { + echo "Installing multicloud/k8s onap4k8s controllers" + if [[ "${testing_enabled}" == "true" ]]; then + echo "Test controllers installation" + for controller_test in sdwan; do + pushd $kud_tests/$controller_test + ansible-playbook $verbose -i \ + $kud_inventory ${controller_test}.yml | \ + tee $cluster_log/test-${controller_test}.log + popd + done + fi +} + +# _print_kubernetes_info() - Prints the login Kubernetes information +function _print_kubernetes_info { + if ! $(kubectl version &>/dev/null); then + return + fi + + # Expose Dashboard using NodePort + node_port=30080 + KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" \ + kubectl -n kube-system edit service kubernetes-dashboard + KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" \ + kubectl -n kube-system edit service kubernetes-dashboard + + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}') + + printf "Kubernetes Info\n===============\n" > $k8s_info_file + echo "Dashboard URL: https:$master_ip:$node_port" >> $k8s_info_file + echo "Admin user: kube" >> $k8s_info_file + echo "Admin password: secret" >> $k8s_info_file +} + +verbose="" +if [[ -n "${KUD_DEBUG:-}" ]]; then + set -o xtrace + verbose="-vvv" +fi + +# Configuration values +dest_folder=/opt +kud_folder=${INSTALLER_DIR} +kud_infra_folder=$kud_folder/../../deployment_infra +kud_playbooks=$kud_infra_folder/playbooks +kud_tests=$kud_folder/../../tests +k8s_info_file=$kud_folder/k8s_info.log +testing_enabled=${KUD_ENABLE_TESTS:-false} + +mkdir -p /opt/csar +export CSAR_DIR=/opt/csar + +function install_pkg { +# Install dependencies + apt-get update + install_prerequisites + install_kubespray +} + +function install_cluster { + install_k8s $1 + if [ ${2:+1} ]; then + echo "install default addons and $2" + install_addons "$2" + else + install_addons + fi + + echo "installed the addons" + if ${KUD_PLUGIN_ENABLED:-false}; then + install_plugin + echo "installed the install_plugin" + install_controllers + echo "installed controllers" + fi + _print_kubernetes_info +} + +function usage { + echo "installer usage:" + echo "./installer.sh --install_pkg - Install the required softwarepackage" + echo "./installer.sh --cluster <cluster name> \ +- Install k8s cluster with default plugins" + echo "./installer.sh --cluster <cluster name> \ +--plugins <plugin_1 plugin_2> - Install k8s cluster with default plugins \ +and additional plugins such as onap4k8s." +} + +if [ $# -eq 0 ]; then + echo "Error: No arguments supplied" + usage + exit 1 +fi + +if [ -z "$1" ]; then + echo "Error: Null argument passed" + usage + exit 1 +fi + +if [ "$1" == "--install_pkg" ]; then + export kud_inventory_folder=$kud_folder/inventory + kud_inventory=$kud_inventory_folder/hosts.ini + install_pkg + echo "install pkg" + exit 0 +fi + +if [ "$1" == "--cluster" ]; then + if [ -z "${2-}" ]; then + echo "Error: Cluster name is null" + usage + exit 1 + fi + + cluster_name=$2 + kud_multi_cluster_path=/opt/kud/multi-cluster + cluster_path=$kud_multi_cluster_path/$cluster_name + echo $cluster_path + if [ ! -d "${cluster_path}" ]; then + echo "Error: cluster_path ${cluster_path} doesn't exit" + usage + exit 1 + fi + + cluster_log=$kud_multi_cluster_path/$cluster_name/log + export kud_inventory_folder=$kud_folder/inventory/$cluster_name + kud_inventory=$kud_inventory_folder/hosts.ini + + mkdir -p $kud_inventory_folder + mkdir -p $cluster_log + cp $kud_multi_cluster_path/$cluster_name/hosts.ini $kud_inventory_folder/ + cp -rf $kud_folder/inventory/group_vars $kud_inventory_folder/ + + if [ ${3:+1} ]; then + if [ "$3" == "--plugins" ]; then + if [ -z "${4-}" ]; then + echo "Error: plugins arguments is null; Refer the usage" + usage + exit 1 + fi + plugins_name=${@:4:$#} + install_cluster $cluster_name "$plugins_name" + exit 0 + else + echo "Error: cluster argument should have plugins; \ + Refer the usage" + usage + exit 1 + fi + fi + install_cluster $cluster_name + exit 0 +fi + +echo "Error: Refer the installer usage" +usage +exit 1 diff --git a/kud/hosting_providers/containerized/inventory/group_vars/all.yml b/kud/hosting_providers/containerized/inventory/group_vars/all.yml new file mode 100644 index 00000000..528430c1 --- /dev/null +++ b/kud/hosting_providers/containerized/inventory/group_vars/all.yml @@ -0,0 +1,2 @@ +kube_log_level: 2 +kubeadm_enabled: true diff --git a/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml new file mode 100644 index 00000000..31d0d669 --- /dev/null +++ b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml @@ -0,0 +1,82 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Kubernetes configuration dirs and system namespace. +# Those are where all the additional config stuff goes +# kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location and namespace. +# Editing those values will almost surely break something. +system_namespace: kube-system + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +kube_api_anonymous_auth: true + +# Users to create for basic auth in Kubernetes API via HTTP +# Optionally add groups for user +kube_api_pwd: "secret" +kube_users: + kube: + pass: "{{kube_api_pwd}}" + role: admin + groups: + - system:masters + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +kube_basic_auth: true +kube_token_auth: true + +# Choose network plugin (calico, contiv, weave or flannel) +# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing +kube_network_plugin: flannel + +# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts +kubeconfig_localhost: true +# Copy kubectl binary on the host that runs Ansible to inventory/artifacts +kubectl_localhost: true + +# Enable MountPropagation gate feature +local_volumes_enabled: true +local_volume_provisioner_enabled: true + +## Change this to use another Kubernetes version, e.g. a current beta release +kube_version: v1.14.3 + +# Helm deployment +helm_enabled: true + +# Kube-proxy proxyMode configuration. +# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and +# works in the kernel space +# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs +#kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: false + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: false + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 + +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 + +# disable localdns cache +enable_nodelocaldns: false diff --git a/kud/hosting_providers/vagrant/Vagrantfile b/kud/hosting_providers/vagrant/Vagrantfile index 2d1b5ab4..58251fe9 100644 --- a/kud/hosting_providers/vagrant/Vagrantfile +++ b/kud/hosting_providers/vagrant/Vagrantfile @@ -10,8 +10,8 @@ ############################################################################## box = { - :virtualbox => { :name => 'elastic/ubuntu-16.04-x86_64', :version => '20180708.0.0' }, - :libvirt => { :name => 'elastic/ubuntu-16.04-x86_64', :version=> '20180210.0.0'} + :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0' }, + :libvirt => { :name => 'peru/ubuntu-18.04-server-amd64'} } require 'yaml' diff --git a/kud/hosting_providers/vagrant/clean_sriov.sh b/kud/hosting_providers/vagrant/clean_sriov.sh new file mode 100644 index 00000000..76b8a960 --- /dev/null +++ b/kud/hosting_providers/vagrant/clean_sriov.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +modprobe -r iavf +kver=`uname -a | awk '{print $3}'` +rm -rf /lib/modules/$kver/updates/drivers/net/ethernet/intel/iavf/iavf.ko +depmod -a +sudo rm -rf /tmp/sriov +sudo rm -rf iavf-3.7.34.tar.gz diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index 41b21f64..235736e1 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -21,6 +21,11 @@ function _install_go { version=$(grep "go_version" ${kud_playbooks}/kud-vars.yml | awk -F "'" '{print $2}') local tarball=go$version.linux-amd64.tar.gz + #gcc is required for go apps compilation + if ! which gcc; then + sudo apt-get install -y gcc + fi + if $(go version &>/dev/null); then return fi @@ -107,6 +112,7 @@ function install_k8s { local_release_dir=$(grep "local_release_dir" $kud_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') local tarball=v$version.tar.gz sudo apt-get install -y sshpass make unzip # install make to run mitogen target and unzip is mitogen playbook dependency + sudo apt-get install -y gnupg2 software-properties-common _install_docker _install_ansible wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball @@ -148,17 +154,20 @@ function install_addons { echo "Installing Kubernetes AddOns" _install_ansible sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors - ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log - if [[ "${testing_enabled}" == "true" ]]; then + done + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov}; do pushd $kud_tests bash ${addon}.sh popd - fi - done + done + fi + echo "Add-ons deployment complete..." } # install_plugin() - Install ONAP Multicloud Kubernetes plugin @@ -229,11 +238,9 @@ kud_playbooks=$kud_infra_folder/playbooks kud_tests=$kud_folder/../../tests k8s_info_file=$kud_folder/k8s_info.log testing_enabled=${KUD_ENABLE_TESTS:-false} - sudo mkdir -p $log_folder sudo mkdir -p /opt/csar sudo chown -R $USER /opt/csar - # Install dependencies # Setup proxy variables if [ -f $kud_folder/sources.list ]; then diff --git a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml index 14146742..fb744d0e 100644 --- a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml +++ b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml @@ -42,7 +42,8 @@ kube_network_plugin: flannel kubeconfig_localhost: true # Copy kubectl binary on the host that runs Ansible to inventory/artifacts kubectl_localhost: true - +# Disable nodelocal dns cache +enable_nodelocaldns: false # Enable MountPropagation gate feature local_volumes_enabled: true local_volume_provisioner_enabled: true @@ -71,3 +72,8 @@ local_release_dir: "/tmp/releases" # cannot access each over via ssh or you want to use local docker # images as a cache for multiple clusters. download_localhost: true + +# Subnet for cluster IPs +kube_service_addresses: 10.244.0.0/18 +# Subnet for Pod IPs +kube_pods_subnet: 10.244.64.0/18 diff --git a/kud/tests/onap4k8s.sh b/kud/tests/onap4k8s.sh new file mode 100755 index 00000000..702bed46 --- /dev/null +++ b/kud/tests/onap4k8s.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail + +source _functions.sh +set +e + +master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}' | awk -F "//" '{print $2}') +onap_svc_node_port=30498 +declare -i timeout=18 +declare -i interval=10 + +base_url="http://$master_ip:$onap_svc_node_port/v1" + +function check_onap_svc { + while ((timeout > 0)); do + echo "try $timeout: Wait for $interval seconds to check for onap svc" + sleep $interval + call_api "$base_url/healthcheck" + call_api_ret=$? + if [[ $call_api_ret -eq 0 ]]; then + echo "onap svc health check is success" + exit 0 + fi + ((timeout-=1)) + done +} + +check_onap_svc +echo "Failed to check for onap svc" +exit 1 diff --git a/kud/tests/plugin_edgex.sh b/kud/tests/plugin_edgex.sh index 8eae5692..ae390add 100755 --- a/kud/tests/plugin_edgex.sh +++ b/kud/tests/plugin_edgex.sh @@ -17,7 +17,16 @@ source _common_test.sh source _functions.sh source _common.sh -base_url="http://localhost:9015/v1" +if [ ${1:+1} ]; then + if [ "$1" == "--external" ]; then + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}' | awk -F "//" '{print $2}') + onap_svc_node_port=30498 + base_url="http://$master_ip:$onap_svc_node_port/v1" + fi +fi + +base_url=${base_url:-"http://localhost:9015/v1"} kubeconfig_path="$HOME/.kube/config" csar_id=cb009bfe-bbee-11e8-9766-525400435678 rb_name="edgex" @@ -91,6 +100,9 @@ response="$(call_api -d "${payload}" "${base_url}/instance")" echo "$response" vnf_id="$(jq -r '.id' <<< "${response}")" +print_msg "Waiting for EdgeX instances" +sleep 240 + print_msg "Validating Kubernetes" kubectl get --no-headers=true --namespace=${namespace} deployment edgex-core-command kubectl get --no-headers=true --namespace=${namespace} service edgex-core-command diff --git a/kud/tests/plugin_fw.sh b/kud/tests/plugin_fw.sh index d7bed4fd..eec467c3 100755 --- a/kud/tests/plugin_fw.sh +++ b/kud/tests/plugin_fw.sh @@ -17,7 +17,16 @@ source _common_test.sh source _functions.sh source _common.sh -base_url="http://localhost:9015/v1" +if [ ${1:+1} ]; then + if [ "$1" == "--external" ]; then + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}' | awk -F "//" '{print $2}') + onap_svc_node_port=30498 + base_url="http://$master_ip:$onap_svc_node_port/v1" + fi +fi + +base_url=${base_url:-"http://localhost:9015/v1"} kubeconfig_path="$HOME/.kube/config" csar_id=cc009bfe-bbee-11e8-9766-525400435678 rb_name="vfw" @@ -98,6 +107,9 @@ wait_for_pod -n "${namespace}" -l app=firewall wait_for_pod -n "${namespace}" -l app=packetgen # TODO: Provide some health check to verify vFW work +print_msg "Waiting for VNF instances" +sleep 480 + print_msg "Retrieving VNF details" call_api "${base_url}/instance/${vnf_id}" diff --git a/kud/tests/sdwan.sh b/kud/tests/sdwan.sh new file mode 100755 index 00000000..64b10f22 --- /dev/null +++ b/kud/tests/sdwan.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +echo "Create pods ..." +kubectl apply -f sdwan/ovn-pod.yml +kubectl apply -f sdwan/sdwan-openwrt-ovn.yml + +bash sdwan/test.sh + +echo "Clear pods ..." +kubectl delete -f sdwan/ovn-pod.yml +kubectl delete -f sdwan/sdwan-openwrt-ovn.yml + +echo "Test Completed!" diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl new file mode 100644 index 00000000..85c7d358 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl @@ -0,0 +1,26 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 +ENV http_proxy={docker_proxy} +ENV https_proxy={docker_proxy} +ENV no_proxy=localhost,120.0.0.1,192.168.* + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +ENV http_proxy= +ENV https_proxy= +ENV no_proxy= + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl new file mode 100644 index 00000000..8b5c57d2 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl @@ -0,0 +1,19 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/README.md b/kud/tests/sdwan/build/README.md new file mode 100644 index 00000000..87e21956 --- /dev/null +++ b/kud/tests/sdwan/build/README.md @@ -0,0 +1,10 @@ +# Introduction: +Please refer ICN SDWAN Module Design for architecture introduction +link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design + +# SDWAN Docker Image build instructions: +Use below steps to build openwrt docker image: openwrt-1806-mwan3 +(1) update set_proxy file with proxy used for docker build +(2) execute build_image.sh +cd build +sudo bash build_image.sh diff --git a/kud/tests/sdwan/build/build_image.sh b/kud/tests/sdwan/build/build_image.sh new file mode 100644 index 00000000..7ff6e20b --- /dev/null +++ b/kud/tests/sdwan/build/build_image.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# usage: build_images.sh + +set -ex +base_image_tag=openwrt-1806-4-base +docker_file=Dockerfile_1806_mwan3 +image_tag=openwrt-1806-mwan3 +package=openwrt-18.06.4-x86-64-generic-rootfs + +# build openwrt base docker images +base_image=`docker images | grep $base_image_tag | awk '{print $1}'` +if [ -z "$base_image" ]; then + # download driver source package + if [ ! -e /tmp/$package.tar.gz ]; then + wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz + fi + cp /tmp/$package.tar.gz . + + docker import $package.tar.gz $base_image_tag +fi + +# generate Dockerfile +test -f ./set_proxy && . set_proxy +docker_proxy=${docker_proxy-""} +if [ -z "$docker_proxy" ]; then + cp ${docker_file}_noproxy.tpl $docker_file +else + cp $docker_file.tpl $docker_file + sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file +fi + +# build docker images for openwrt with wman3 +docker build --network=host -f $docker_file -t $image_tag . + +# clear +docker image rm $base_image_tag +rm -rf $docker_file +rm -rf $package.tar.gz diff --git a/kud/tests/sdwan/build/commands.lua b/kud/tests/sdwan/build/commands.lua new file mode 100644 index 00000000..d99f4579 --- /dev/null +++ b/kud/tests/sdwan/build/commands.lua @@ -0,0 +1,43 @@ +-- Licensed to the public under the GNU General Public License v2. + +module("luci.controller.commands", package.seeall) + +sys = require "luci.sys" +ut = require "luci.util" +io = require "io" + +ip = "ip -4 " + +function index() + entry({"admin", "config", "command"}, + call("execute")).dependent = false +end + +function trim(s) + return s:match("^%s*(.-)%s*$") +end + +function split_and_trim(str, sep) + local array = {} + local reg = string.format("([^%s]+)", sep) + for item in string.gmatch(str, reg) do + item_trimed = trim(item) + if string.len(item_trimed) > 0 then + table.insert(array, item_trimed) + end + end + return array +end + +function execute() + local commands = luci.http.formvalue("command") + io.stderr:write("Execute command: %s\n" % commands) + + local command_array = split_and_trim(commands, ";") + for index, command in ipairs(command_array) do + sys.exec(command) + end + + luci.http.prepare_content("application/json") + luci.http.write_json("{'status':'ok'}") +end diff --git a/kud/tests/sdwan/build/set_proxy b/kud/tests/sdwan/build/set_proxy new file mode 100644 index 00000000..7a195fe5 --- /dev/null +++ b/kud/tests/sdwan/build/set_proxy @@ -0,0 +1,2 @@ +# set docker proxy with below line, the build script will use this info +#docker_proxy= diff --git a/kud/tests/sdwan/build/system b/kud/tests/sdwan/build/system new file mode 100644 index 00000000..5165430f --- /dev/null +++ b/kud/tests/sdwan/build/system @@ -0,0 +1,7 @@ +config system + option log_file '/var/log/mylog' + option timezone 'UTC' + option ttylogin '0' + option log_size '64' + option urandom_seed '0' +EOF diff --git a/kud/tests/sdwan/ovn-pod.yml b/kud/tests/sdwan/ovn-pod.yml new file mode 100644 index 00000000..0715c030 --- /dev/null +++ b/kud/tests/sdwan/ovn-pod.yml @@ -0,0 +1,40 @@ +# Create 2 ovn4nfv network attachment definition +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-port-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.33.0/24 + name: subnet1 + gateway: 172.16.33.1/24 + +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-priv-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.44.0/24 + name: subnet1 + gateway: 172.16.44.1/24 + +--- +apiVersion: v1 +kind: Pod +metadata: + name: ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: ovn-pod + image: docker.io/centos/tools:latest + command: + - /sbin/init diff --git a/kud/tests/sdwan/sdwan-openwrt-ovn.yml b/kud/tests/sdwan/sdwan-openwrt-ovn.yml new file mode 100644 index 00000000..2accdc6c --- /dev/null +++ b/kud/tests/sdwan/sdwan-openwrt-ovn.yml @@ -0,0 +1,82 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sdwan-config-ovn +data: + entrypoint.sh: | + #!/bin/bash + # Always exit on errors. + set -e + + interface0=net0 + ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + interface1=net1 + ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + net_config=/etc/config/network + cat >> $net_config << EOF + config interface 'wan' + option ifname '$interface0' + option proto 'static' + option ipaddr '$ipaddr0' + option netmask '255.255.255.0' + + config interface 'wanb' + option ifname '$interface1' + option proto 'static' + option ipaddr '$ipaddr1' + option netmask '255.255.255.0' + EOF + + /sbin/procd & + /sbin/ubusd & + iptables -S + sleep 1 + /etc/init.d/rpcd start + /etc/init.d/dnsmasq start + /etc/init.d/network start + /etc/init.d/odhcpd start + /etc/init.d/uhttpd start + /etc/init.d/log start + /etc/init.d/dropbear start + /etc/init.d/mwan3 restart + + echo "Entering sleep... (success)" + + # Sleep forever. + while true; do sleep 100; done + +--- +apiVersion: v1 +kind: Pod +metadata: + name: sdwan-ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: sdwan-ovn-pod + image: hle2/openwrt-1806-mwan3:v0.1.0 + ports: + - containerPort: 22 + - containerPort: 80 + command: + - /bin/sh + - /init/entrypoint.sh + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: entrypoint-sh + mountPath: /init + volumes: + - name: entrypoint-sh + configMap: + name: sdwan-config-ovn + items: + - key: entrypoint.sh + path: entrypoint.sh diff --git a/kud/tests/sdwan/sdwan.yml b/kud/tests/sdwan/sdwan.yml new file mode 100644 index 00000000..760d8599 --- /dev/null +++ b/kud/tests/sdwan/sdwan.yml @@ -0,0 +1,44 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +- hosts: localhost + become: yes + tasks: + - name: create ovn network and client workload + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/ovn-pod.yml" + + - name: create sdwan controller + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" + +- hosts: kube-master + become: yes + tasks: + - name: install wget package for ubuntu systems + apt: name=wget state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install wget package for centos systems + yum: name=wget state=present update_cache=yes + when: ansible_distribution == "CentOS" + + - name: Execute sdwan test script in cluster master + script: test.sh + register: sdwan + + - debug: + var: sdwan.stdout_lines + +- hosts: localhost + become: yes + tasks: + - name: delete ovn network and client workload + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/ovn-pod.yml" + + - name: delete sdwan controller + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" diff --git a/kud/tests/sdwan/test.sh b/kud/tests/sdwan/test.sh new file mode 100755 index 00000000..ba4b4173 --- /dev/null +++ b/kud/tests/sdwan/test.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +sdwan_pod_name=sdwan-ovn-pod +ovn_pod_name=ovn-pod +wan_interface=net0 + +function login { + login_url=http://$1/cgi-bin/luci/ + echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/') +} + +function disable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='REJECT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function enable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function wait_for_pod { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +function wait_for_pod_namespace { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +echo "Waiting for pods to be ready ..." +wait_for_pod $ovn_pod_name +wait_for_pod $sdwan_pod_name +echo "* Create pods success" + +sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}') +ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}') +echo "SDWAN pod ip:"$sdwan_pod_ip +echo "OVN pod ip:"$ovn_pod_ip + +echo "Login to sdwan ..." +security_token="" +while [[ "$security_token" == "" ]]; do + echo "Get Security Token ..." + security_token=$(login $sdwan_pod_ip) + sleep 2 +done +echo "* Security Token: "$security_token + +kubectl exec $sdwan_pod_name ifconfig + +sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}') +echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is work through wan interface" +else + echo "* Test failed!" + exit 1 +fi + +echo "Disable ping rule of wan interface ..." +ret=$(disable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is not work through wan interface after ping rule disabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true) +if [[ $ping_result == *", 100% packet loss"* ]]; then + echo "* Ping is disabled" +else + echo "* Test failed!" + exit 1 +fi + +echo "Enable ping rule of wan interface ..." +ret=$(enable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is work through wan interface after ping rule enabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is enabled" +else + echo "* Test failed!" + exit 1 +fi + + +echo "Test Completed!" diff --git a/kud/tests/sriov.sh b/kud/tests/sriov.sh new file mode 100755 index 00000000..a721b722 --- /dev/null +++ b/kud/tests/sriov.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o pipefail + +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) +if [ -z "$ethernet_adpator_version" ]; then + echo " Ethernet adapator version is not set. SRIOV test case cannot run on this machine" + exit 0 +fi +#checking for the right hardware version of NIC on the machine +if [ $ethernet_adpator_version == "XL710" ]; then + echo "NIC card specs match. SRIOV option avaiable for this version." +else + echo -e "Failed. The version supplied does not match.\nTest cannot be executed." + exit 0 +fi + +pod_name=pod-case-01 +rm -f $HOME/$pod_name.yaml +kubectl delete pod $pod_name --ignore-not-found=true --now --wait +allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_700" | tail -n1 |awk '{print $(NF)}') + +echo "The allocated resource of the node is: " $allocated_node_resource +cat << POD > $HOME/$pod_name.yaml +apiVersion: v1 +kind: Pod +metadata: + name: pod-case-01 + annotations: + k8s.v1.cni.cncf.io/networks: sriov-eno2 +spec: + containers: + - name: test-pod + image: docker.io/centos/tools:latest + command: + - /sbin/init + resources: + requests: + intel.com/intel_sriov_700: '1' + limits: + intel.com/intel_sriov_700: '1' +POD +kubectl create -f $HOME/$pod_name.yaml --validate=false + for pod in $pod_name; do + status_phase="" + while [[ $status_phase != "Running" ]]; do + new_phase=$(kubectl get pods $pod | awk 'NR==2{print $3}') + if [[ $new_phase != $status_phase ]]; then + echo "$(date +%H:%M:%S) - $pod : $new_phase" + status_phase=$new_phase + fi + if [[ $new_phase == "Running" ]]; then + echo "Pod is up and running.." + fi + if [[ $new_phase == "Err"* ]]; then + exit 1 + fi + done + done +allocated_node_resource=$(kubectl describe node | grep "intel.com/intel_sriov_700" | tail -n1 |awk '{print $(NF)}') + +echo " The current resource allocation after the pod creation is: " $allocated_node_resource +kubectl delete pod $pod_name --now +echo "Test complete." |