From b86512e598d5e59fe1f2048159e68dea4d229164 Mon Sep 17 00:00:00 2001 From: "r.kuralamudhan" Date: Fri, 22 Nov 2019 13:20:52 -0800 Subject: Adding SDWAN testing for KUD containerized installer script Please refer ICN SDWAN Module Design for architecture link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design Issue-ID: MULTICLOUD-956 Co-authored-by: Huifeng Le Signed-off-by: r.kuralamudhan Change-Id: I4bc35bc62f6bab52a5d290829f7406424d72d5ae --- kud/hosting_providers/containerized/installer.sh | 17 +++ kud/tests/sdwan.sh | 25 +++++ kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl | 26 +++++ .../sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl | 19 ++++ kud/tests/sdwan/build/README.md | 10 ++ kud/tests/sdwan/build/build_image.sh | 39 +++++++ kud/tests/sdwan/build/commands.lua | 43 ++++++++ kud/tests/sdwan/build/set_proxy | 2 + kud/tests/sdwan/build/system | 7 ++ kud/tests/sdwan/ovn-pod.yml | 40 +++++++ kud/tests/sdwan/sdwan-openwrt-ovn.yml | 82 ++++++++++++++ kud/tests/sdwan/sdwan.yml | 44 ++++++++ kud/tests/sdwan/test.sh | 120 +++++++++++++++++++++ 13 files changed, 474 insertions(+) create mode 100755 kud/tests/sdwan.sh create mode 100644 kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl create mode 100644 kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl create mode 100644 kud/tests/sdwan/build/README.md create mode 100644 kud/tests/sdwan/build/build_image.sh create mode 100644 kud/tests/sdwan/build/commands.lua create mode 100644 kud/tests/sdwan/build/set_proxy create mode 100644 kud/tests/sdwan/build/system create mode 100644 kud/tests/sdwan/ovn-pod.yml create mode 100644 kud/tests/sdwan/sdwan-openwrt-ovn.yml create mode 100644 kud/tests/sdwan/sdwan.yml create mode 100755 kud/tests/sdwan/test.sh diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 713847ea..f1b95acb 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -149,6 +149,21 @@ function install_plugin { fi } +# install_controllers() - Install ONAP Multicloud Kubernetes controllers +function install_controllers { + echo "Installing multicloud/k8s onap4k8s controllers" + if [[ "${testing_enabled}" == "true" ]]; then + echo "Test controllers installation" + for controller_test in sdwan; do + pushd $kud_tests/$controller_test + ansible-playbook $verbose -i \ + $kud_inventory ${controller_test}.yml | \ + tee $cluster_log/test-${controller_test}.log + popd + done + fi +} + # _print_kubernetes_info() - Prints the login Kubernetes information function _print_kubernetes_info { if ! $(kubectl version &>/dev/null); then @@ -209,6 +224,8 @@ function install_cluster { if ${KUD_PLUGIN_ENABLED:-false}; then install_plugin echo "installed the install_plugin" + install_controllers + echo "installed controllers" fi _print_kubernetes_info } diff --git a/kud/tests/sdwan.sh b/kud/tests/sdwan.sh new file mode 100755 index 00000000..64b10f22 --- /dev/null +++ b/kud/tests/sdwan.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +echo "Create pods ..." +kubectl apply -f sdwan/ovn-pod.yml +kubectl apply -f sdwan/sdwan-openwrt-ovn.yml + +bash sdwan/test.sh + +echo "Clear pods ..." +kubectl delete -f sdwan/ovn-pod.yml +kubectl delete -f sdwan/sdwan-openwrt-ovn.yml + +echo "Test Completed!" diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl new file mode 100644 index 00000000..85c7d358 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl @@ -0,0 +1,26 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 +ENV http_proxy={docker_proxy} +ENV https_proxy={docker_proxy} +ENV no_proxy=localhost,120.0.0.1,192.168.* + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +ENV http_proxy= +ENV https_proxy= +ENV no_proxy= + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl new file mode 100644 index 00000000..8b5c57d2 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl @@ -0,0 +1,19 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/README.md b/kud/tests/sdwan/build/README.md new file mode 100644 index 00000000..87e21956 --- /dev/null +++ b/kud/tests/sdwan/build/README.md @@ -0,0 +1,10 @@ +# Introduction: +Please refer ICN SDWAN Module Design for architecture introduction +link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design + +# SDWAN Docker Image build instructions: +Use below steps to build openwrt docker image: openwrt-1806-mwan3 +(1) update set_proxy file with proxy used for docker build +(2) execute build_image.sh +cd build +sudo bash build_image.sh diff --git a/kud/tests/sdwan/build/build_image.sh b/kud/tests/sdwan/build/build_image.sh new file mode 100644 index 00000000..7ff6e20b --- /dev/null +++ b/kud/tests/sdwan/build/build_image.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# usage: build_images.sh + +set -ex +base_image_tag=openwrt-1806-4-base +docker_file=Dockerfile_1806_mwan3 +image_tag=openwrt-1806-mwan3 +package=openwrt-18.06.4-x86-64-generic-rootfs + +# build openwrt base docker images +base_image=`docker images | grep $base_image_tag | awk '{print $1}'` +if [ -z "$base_image" ]; then + # download driver source package + if [ ! -e /tmp/$package.tar.gz ]; then + wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz + fi + cp /tmp/$package.tar.gz . + + docker import $package.tar.gz $base_image_tag +fi + +# generate Dockerfile +test -f ./set_proxy && . set_proxy +docker_proxy=${docker_proxy-""} +if [ -z "$docker_proxy" ]; then + cp ${docker_file}_noproxy.tpl $docker_file +else + cp $docker_file.tpl $docker_file + sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file +fi + +# build docker images for openwrt with wman3 +docker build --network=host -f $docker_file -t $image_tag . + +# clear +docker image rm $base_image_tag +rm -rf $docker_file +rm -rf $package.tar.gz diff --git a/kud/tests/sdwan/build/commands.lua b/kud/tests/sdwan/build/commands.lua new file mode 100644 index 00000000..d99f4579 --- /dev/null +++ b/kud/tests/sdwan/build/commands.lua @@ -0,0 +1,43 @@ +-- Licensed to the public under the GNU General Public License v2. + +module("luci.controller.commands", package.seeall) + +sys = require "luci.sys" +ut = require "luci.util" +io = require "io" + +ip = "ip -4 " + +function index() + entry({"admin", "config", "command"}, + call("execute")).dependent = false +end + +function trim(s) + return s:match("^%s*(.-)%s*$") +end + +function split_and_trim(str, sep) + local array = {} + local reg = string.format("([^%s]+)", sep) + for item in string.gmatch(str, reg) do + item_trimed = trim(item) + if string.len(item_trimed) > 0 then + table.insert(array, item_trimed) + end + end + return array +end + +function execute() + local commands = luci.http.formvalue("command") + io.stderr:write("Execute command: %s\n" % commands) + + local command_array = split_and_trim(commands, ";") + for index, command in ipairs(command_array) do + sys.exec(command) + end + + luci.http.prepare_content("application/json") + luci.http.write_json("{'status':'ok'}") +end diff --git a/kud/tests/sdwan/build/set_proxy b/kud/tests/sdwan/build/set_proxy new file mode 100644 index 00000000..7a195fe5 --- /dev/null +++ b/kud/tests/sdwan/build/set_proxy @@ -0,0 +1,2 @@ +# set docker proxy with below line, the build script will use this info +#docker_proxy= diff --git a/kud/tests/sdwan/build/system b/kud/tests/sdwan/build/system new file mode 100644 index 00000000..5165430f --- /dev/null +++ b/kud/tests/sdwan/build/system @@ -0,0 +1,7 @@ +config system + option log_file '/var/log/mylog' + option timezone 'UTC' + option ttylogin '0' + option log_size '64' + option urandom_seed '0' +EOF diff --git a/kud/tests/sdwan/ovn-pod.yml b/kud/tests/sdwan/ovn-pod.yml new file mode 100644 index 00000000..0715c030 --- /dev/null +++ b/kud/tests/sdwan/ovn-pod.yml @@ -0,0 +1,40 @@ +# Create 2 ovn4nfv network attachment definition +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-port-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.33.0/24 + name: subnet1 + gateway: 172.16.33.1/24 + +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-priv-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.44.0/24 + name: subnet1 + gateway: 172.16.44.1/24 + +--- +apiVersion: v1 +kind: Pod +metadata: + name: ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: ovn-pod + image: docker.io/centos/tools:latest + command: + - /sbin/init diff --git a/kud/tests/sdwan/sdwan-openwrt-ovn.yml b/kud/tests/sdwan/sdwan-openwrt-ovn.yml new file mode 100644 index 00000000..2accdc6c --- /dev/null +++ b/kud/tests/sdwan/sdwan-openwrt-ovn.yml @@ -0,0 +1,82 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sdwan-config-ovn +data: + entrypoint.sh: | + #!/bin/bash + # Always exit on errors. + set -e + + interface0=net0 + ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + interface1=net1 + ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + net_config=/etc/config/network + cat >> $net_config << EOF + config interface 'wan' + option ifname '$interface0' + option proto 'static' + option ipaddr '$ipaddr0' + option netmask '255.255.255.0' + + config interface 'wanb' + option ifname '$interface1' + option proto 'static' + option ipaddr '$ipaddr1' + option netmask '255.255.255.0' + EOF + + /sbin/procd & + /sbin/ubusd & + iptables -S + sleep 1 + /etc/init.d/rpcd start + /etc/init.d/dnsmasq start + /etc/init.d/network start + /etc/init.d/odhcpd start + /etc/init.d/uhttpd start + /etc/init.d/log start + /etc/init.d/dropbear start + /etc/init.d/mwan3 restart + + echo "Entering sleep... (success)" + + # Sleep forever. + while true; do sleep 100; done + +--- +apiVersion: v1 +kind: Pod +metadata: + name: sdwan-ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: sdwan-ovn-pod + image: hle2/openwrt-1806-mwan3:v0.1.0 + ports: + - containerPort: 22 + - containerPort: 80 + command: + - /bin/sh + - /init/entrypoint.sh + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: entrypoint-sh + mountPath: /init + volumes: + - name: entrypoint-sh + configMap: + name: sdwan-config-ovn + items: + - key: entrypoint.sh + path: entrypoint.sh diff --git a/kud/tests/sdwan/sdwan.yml b/kud/tests/sdwan/sdwan.yml new file mode 100644 index 00000000..760d8599 --- /dev/null +++ b/kud/tests/sdwan/sdwan.yml @@ -0,0 +1,44 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +- hosts: localhost + become: yes + tasks: + - name: create ovn network and client workload + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/ovn-pod.yml" + + - name: create sdwan controller + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" + +- hosts: kube-master + become: yes + tasks: + - name: install wget package for ubuntu systems + apt: name=wget state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install wget package for centos systems + yum: name=wget state=present update_cache=yes + when: ansible_distribution == "CentOS" + + - name: Execute sdwan test script in cluster master + script: test.sh + register: sdwan + + - debug: + var: sdwan.stdout_lines + +- hosts: localhost + become: yes + tasks: + - name: delete ovn network and client workload + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/ovn-pod.yml" + + - name: delete sdwan controller + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" diff --git a/kud/tests/sdwan/test.sh b/kud/tests/sdwan/test.sh new file mode 100755 index 00000000..ba4b4173 --- /dev/null +++ b/kud/tests/sdwan/test.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +sdwan_pod_name=sdwan-ovn-pod +ovn_pod_name=ovn-pod +wan_interface=net0 + +function login { + login_url=http://$1/cgi-bin/luci/ + echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/') +} + +function disable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='REJECT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function enable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function wait_for_pod { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +function wait_for_pod_namespace { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +echo "Waiting for pods to be ready ..." +wait_for_pod $ovn_pod_name +wait_for_pod $sdwan_pod_name +echo "* Create pods success" + +sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}') +ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}') +echo "SDWAN pod ip:"$sdwan_pod_ip +echo "OVN pod ip:"$ovn_pod_ip + +echo "Login to sdwan ..." +security_token="" +while [[ "$security_token" == "" ]]; do + echo "Get Security Token ..." + security_token=$(login $sdwan_pod_ip) + sleep 2 +done +echo "* Security Token: "$security_token + +kubectl exec $sdwan_pod_name ifconfig + +sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}') +echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is work through wan interface" +else + echo "* Test failed!" + exit 1 +fi + +echo "Disable ping rule of wan interface ..." +ret=$(disable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is not work through wan interface after ping rule disabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true) +if [[ $ping_result == *", 100% packet loss"* ]]; then + echo "* Ping is disabled" +else + echo "* Test failed!" + exit 1 +fi + +echo "Enable ping rule of wan interface ..." +ret=$(enable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is work through wan interface after ping rule enabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is enabled" +else + echo "* Test failed!" + exit 1 +fi + + +echo "Test Completed!" -- cgit 1.2.3-korg