aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorr.kuralamudhan <kuralamudhan.ramakrishnan@intel.com>2019-11-22 13:20:52 -0800
committerr.kuralamudhan <kuralamudhan.ramakrishnan@intel.com>2019-11-22 16:20:35 -0800
commitb86512e598d5e59fe1f2048159e68dea4d229164 (patch)
treed585098207df9fc477bb895b399767bd508ff5e5
parentdfdc0206daee4b3474072e2fdd6bc39ab77af5c0 (diff)
Adding SDWAN testing for KUD containerized installer script
Please refer ICN SDWAN Module Design for architecture link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design Issue-ID: MULTICLOUD-956 Co-authored-by: Huifeng Le <huifeng.le@intel.com> Signed-off-by: r.kuralamudhan <kuralamudhan.ramakrishnan@intel.com> Change-Id: I4bc35bc62f6bab52a5d290829f7406424d72d5ae
-rwxr-xr-xkud/hosting_providers/containerized/installer.sh17
-rwxr-xr-xkud/tests/sdwan.sh25
-rw-r--r--kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl26
-rw-r--r--kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl19
-rw-r--r--kud/tests/sdwan/build/README.md10
-rw-r--r--kud/tests/sdwan/build/build_image.sh39
-rw-r--r--kud/tests/sdwan/build/commands.lua43
-rw-r--r--kud/tests/sdwan/build/set_proxy2
-rw-r--r--kud/tests/sdwan/build/system7
-rw-r--r--kud/tests/sdwan/ovn-pod.yml40
-rw-r--r--kud/tests/sdwan/sdwan-openwrt-ovn.yml82
-rw-r--r--kud/tests/sdwan/sdwan.yml44
-rwxr-xr-xkud/tests/sdwan/test.sh120
13 files changed, 474 insertions, 0 deletions
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index 713847ea..f1b95acb 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -149,6 +149,21 @@ function install_plugin {
fi
}
+# install_controllers() - Install ONAP Multicloud Kubernetes controllers
+function install_controllers {
+ echo "Installing multicloud/k8s onap4k8s controllers"
+ if [[ "${testing_enabled}" == "true" ]]; then
+ echo "Test controllers installation"
+ for controller_test in sdwan; do
+ pushd $kud_tests/$controller_test
+ ansible-playbook $verbose -i \
+ $kud_inventory ${controller_test}.yml | \
+ tee $cluster_log/test-${controller_test}.log
+ popd
+ done
+ fi
+}
+
# _print_kubernetes_info() - Prints the login Kubernetes information
function _print_kubernetes_info {
if ! $(kubectl version &>/dev/null); then
@@ -209,6 +224,8 @@ function install_cluster {
if ${KUD_PLUGIN_ENABLED:-false}; then
install_plugin
echo "installed the install_plugin"
+ install_controllers
+ echo "installed controllers"
fi
_print_kubernetes_info
}
diff --git a/kud/tests/sdwan.sh b/kud/tests/sdwan.sh
new file mode 100755
index 00000000..64b10f22
--- /dev/null
+++ b/kud/tests/sdwan.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Create pods ..."
+kubectl apply -f sdwan/ovn-pod.yml
+kubectl apply -f sdwan/sdwan-openwrt-ovn.yml
+
+bash sdwan/test.sh
+
+echo "Clear pods ..."
+kubectl delete -f sdwan/ovn-pod.yml
+kubectl delete -f sdwan/sdwan-openwrt-ovn.yml
+
+echo "Test Completed!"
diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl
new file mode 100644
index 00000000..85c7d358
--- /dev/null
+++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl
@@ -0,0 +1,26 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+ENV http_proxy={docker_proxy}
+ENV https_proxy={docker_proxy}
+ENV no_proxy=localhost,120.0.0.1,192.168.*
+
+RUN mkdir /var/lock && \
+ opkg update && \
+ opkg install uhttpd-mod-lua && \
+ uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+ uci commit uhttpd && \
+ opkg install mwan3 && \
+ opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+ENV http_proxy=
+ENV https_proxy=
+ENV no_proxy=
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
new file mode 100644
index 00000000..8b5c57d2
--- /dev/null
+++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
@@ -0,0 +1,19 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+
+RUN mkdir /var/lock && \
+ opkg update && \
+ opkg install uhttpd-mod-lua && \
+ uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+ uci commit uhttpd && \
+ opkg install mwan3 && \
+ opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/kud/tests/sdwan/build/README.md b/kud/tests/sdwan/build/README.md
new file mode 100644
index 00000000..87e21956
--- /dev/null
+++ b/kud/tests/sdwan/build/README.md
@@ -0,0 +1,10 @@
+# Introduction:
+Please refer ICN SDWAN Module Design for architecture introduction
+link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design
+
+# SDWAN Docker Image build instructions:
+Use below steps to build openwrt docker image: openwrt-1806-mwan3
+(1) update set_proxy file with proxy used for docker build
+(2) execute build_image.sh
+cd build
+sudo bash build_image.sh
diff --git a/kud/tests/sdwan/build/build_image.sh b/kud/tests/sdwan/build/build_image.sh
new file mode 100644
index 00000000..7ff6e20b
--- /dev/null
+++ b/kud/tests/sdwan/build/build_image.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# usage: build_images.sh
+
+set -ex
+base_image_tag=openwrt-1806-4-base
+docker_file=Dockerfile_1806_mwan3
+image_tag=openwrt-1806-mwan3
+package=openwrt-18.06.4-x86-64-generic-rootfs
+
+# build openwrt base docker images
+base_image=`docker images | grep $base_image_tag | awk '{print $1}'`
+if [ -z "$base_image" ]; then
+ # download driver source package
+ if [ ! -e /tmp/$package.tar.gz ]; then
+ wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz
+ fi
+ cp /tmp/$package.tar.gz .
+
+ docker import $package.tar.gz $base_image_tag
+fi
+
+# generate Dockerfile
+test -f ./set_proxy && . set_proxy
+docker_proxy=${docker_proxy-""}
+if [ -z "$docker_proxy" ]; then
+ cp ${docker_file}_noproxy.tpl $docker_file
+else
+ cp $docker_file.tpl $docker_file
+ sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file
+fi
+
+# build docker images for openwrt with wman3
+docker build --network=host -f $docker_file -t $image_tag .
+
+# clear
+docker image rm $base_image_tag
+rm -rf $docker_file
+rm -rf $package.tar.gz
diff --git a/kud/tests/sdwan/build/commands.lua b/kud/tests/sdwan/build/commands.lua
new file mode 100644
index 00000000..d99f4579
--- /dev/null
+++ b/kud/tests/sdwan/build/commands.lua
@@ -0,0 +1,43 @@
+-- Licensed to the public under the GNU General Public License v2.
+
+module("luci.controller.commands", package.seeall)
+
+sys = require "luci.sys"
+ut = require "luci.util"
+io = require "io"
+
+ip = "ip -4 "
+
+function index()
+ entry({"admin", "config", "command"},
+ call("execute")).dependent = false
+end
+
+function trim(s)
+ return s:match("^%s*(.-)%s*$")
+end
+
+function split_and_trim(str, sep)
+ local array = {}
+ local reg = string.format("([^%s]+)", sep)
+ for item in string.gmatch(str, reg) do
+ item_trimed = trim(item)
+ if string.len(item_trimed) > 0 then
+ table.insert(array, item_trimed)
+ end
+ end
+ return array
+end
+
+function execute()
+ local commands = luci.http.formvalue("command")
+ io.stderr:write("Execute command: %s\n" % commands)
+
+ local command_array = split_and_trim(commands, ";")
+ for index, command in ipairs(command_array) do
+ sys.exec(command)
+ end
+
+ luci.http.prepare_content("application/json")
+ luci.http.write_json("{'status':'ok'}")
+end
diff --git a/kud/tests/sdwan/build/set_proxy b/kud/tests/sdwan/build/set_proxy
new file mode 100644
index 00000000..7a195fe5
--- /dev/null
+++ b/kud/tests/sdwan/build/set_proxy
@@ -0,0 +1,2 @@
+# set docker proxy with below line, the build script will use this info
+#docker_proxy=
diff --git a/kud/tests/sdwan/build/system b/kud/tests/sdwan/build/system
new file mode 100644
index 00000000..5165430f
--- /dev/null
+++ b/kud/tests/sdwan/build/system
@@ -0,0 +1,7 @@
+config system
+ option log_file '/var/log/mylog'
+ option timezone 'UTC'
+ option ttylogin '0'
+ option log_size '64'
+ option urandom_seed '0'
+EOF
diff --git a/kud/tests/sdwan/ovn-pod.yml b/kud/tests/sdwan/ovn-pod.yml
new file mode 100644
index 00000000..0715c030
--- /dev/null
+++ b/kud/tests/sdwan/ovn-pod.yml
@@ -0,0 +1,40 @@
+# Create 2 ovn4nfv network attachment definition
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+ name: ovn-port-net
+spec:
+ cniType : ovn4nfv
+ ipv4Subnets:
+ - subnet: 172.16.33.0/24
+ name: subnet1
+ gateway: 172.16.33.1/24
+
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+ name: ovn-priv-net
+spec:
+ cniType : ovn4nfv
+ ipv4Subnets:
+ - subnet: 172.16.44.0/24
+ name: subnet1
+ gateway: 172.16.44.1/24
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: ovn-pod
+ annotations:
+ k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+ k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+ { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+ containers:
+ - name: ovn-pod
+ image: docker.io/centos/tools:latest
+ command:
+ - /sbin/init
diff --git a/kud/tests/sdwan/sdwan-openwrt-ovn.yml b/kud/tests/sdwan/sdwan-openwrt-ovn.yml
new file mode 100644
index 00000000..2accdc6c
--- /dev/null
+++ b/kud/tests/sdwan/sdwan-openwrt-ovn.yml
@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sdwan-config-ovn
+data:
+ entrypoint.sh: |
+ #!/bin/bash
+ # Always exit on errors.
+ set -e
+
+ interface0=net0
+ ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+ interface1=net1
+ ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+ net_config=/etc/config/network
+ cat >> $net_config << EOF
+ config interface 'wan'
+ option ifname '$interface0'
+ option proto 'static'
+ option ipaddr '$ipaddr0'
+ option netmask '255.255.255.0'
+
+ config interface 'wanb'
+ option ifname '$interface1'
+ option proto 'static'
+ option ipaddr '$ipaddr1'
+ option netmask '255.255.255.0'
+ EOF
+
+ /sbin/procd &
+ /sbin/ubusd &
+ iptables -S
+ sleep 1
+ /etc/init.d/rpcd start
+ /etc/init.d/dnsmasq start
+ /etc/init.d/network start
+ /etc/init.d/odhcpd start
+ /etc/init.d/uhttpd start
+ /etc/init.d/log start
+ /etc/init.d/dropbear start
+ /etc/init.d/mwan3 restart
+
+ echo "Entering sleep... (success)"
+
+ # Sleep forever.
+ while true; do sleep 100; done
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: sdwan-ovn-pod
+ annotations:
+ k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+ k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+ { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+ containers:
+ - name: sdwan-ovn-pod
+ image: hle2/openwrt-1806-mwan3:v0.1.0
+ ports:
+ - containerPort: 22
+ - containerPort: 80
+ command:
+ - /bin/sh
+ - /init/entrypoint.sh
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: entrypoint-sh
+ mountPath: /init
+ volumes:
+ - name: entrypoint-sh
+ configMap:
+ name: sdwan-config-ovn
+ items:
+ - key: entrypoint.sh
+ path: entrypoint.sh
diff --git a/kud/tests/sdwan/sdwan.yml b/kud/tests/sdwan/sdwan.yml
new file mode 100644
index 00000000..760d8599
--- /dev/null
+++ b/kud/tests/sdwan/sdwan.yml
@@ -0,0 +1,44 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+- hosts: localhost
+ become: yes
+ tasks:
+ - name: create ovn network and client workload
+ command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/ovn-pod.yml"
+
+ - name: create sdwan controller
+ command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml"
+
+- hosts: kube-master
+ become: yes
+ tasks:
+ - name: install wget package for ubuntu systems
+ apt: name=wget state=present update_cache=yes
+ when: ansible_distribution == "Ubuntu"
+
+ - name: install wget package for centos systems
+ yum: name=wget state=present update_cache=yes
+ when: ansible_distribution == "CentOS"
+
+ - name: Execute sdwan test script in cluster master
+ script: test.sh
+ register: sdwan
+
+ - debug:
+ var: sdwan.stdout_lines
+
+- hosts: localhost
+ become: yes
+ tasks:
+ - name: delete ovn network and client workload
+ command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/ovn-pod.yml"
+
+ - name: delete sdwan controller
+ command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml"
diff --git a/kud/tests/sdwan/test.sh b/kud/tests/sdwan/test.sh
new file mode 100755
index 00000000..ba4b4173
--- /dev/null
+++ b/kud/tests/sdwan/test.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+sdwan_pod_name=sdwan-ovn-pod
+ovn_pod_name=ovn-pod
+wan_interface=net0
+
+function login {
+ login_url=http://$1/cgi-bin/luci/
+ echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/')
+}
+
+function disable_ping {
+ command_url=http://$2/cgi-bin/luci/admin/config/command
+ command="uci set firewall.@rule[1].target='REJECT';fw3 reload"
+ echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function enable_ping {
+ command_url=http://$2/cgi-bin/luci/admin/config/command
+ command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload"
+ echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function wait_for_pod {
+ status_phase=""
+ while [[ "$status_phase" != "Running" ]]; do
+ new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')"
+ if [[ "$new_phase" != "$status_phase" ]]; then
+ status_phase="$new_phase"
+ fi
+ if [[ "$new_phase" == "Err"* ]]; then
+ exit 1
+ fi
+ sleep 2
+ done
+}
+
+function wait_for_pod_namespace {
+ status_phase=""
+ while [[ "$status_phase" != "Running" ]]; do
+ new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')"
+ if [[ "$new_phase" != "$status_phase" ]]; then
+ status_phase="$new_phase"
+ fi
+ if [[ "$new_phase" == "Err"* ]]; then
+ exit 1
+ fi
+ sleep 2
+ done
+}
+
+echo "Waiting for pods to be ready ..."
+wait_for_pod $ovn_pod_name
+wait_for_pod $sdwan_pod_name
+echo "* Create pods success"
+
+sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}')
+ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}')
+echo "SDWAN pod ip:"$sdwan_pod_ip
+echo "OVN pod ip:"$ovn_pod_ip
+
+echo "Login to sdwan ..."
+security_token=""
+while [[ "$security_token" == "" ]]; do
+ echo "Get Security Token ..."
+ security_token=$(login $sdwan_pod_ip)
+ sleep 2
+done
+echo "* Security Token: "$security_token
+
+kubectl exec $sdwan_pod_name ifconfig
+
+sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}')
+echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+ echo "* Ping is work through wan interface"
+else
+ echo "* Test failed!"
+ exit 1
+fi
+
+echo "Disable ping rule of wan interface ..."
+ret=$(disable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is not work through wan interface after ping rule disabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true)
+if [[ $ping_result == *", 100% packet loss"* ]]; then
+ echo "* Ping is disabled"
+else
+ echo "* Test failed!"
+ exit 1
+fi
+
+echo "Enable ping rule of wan interface ..."
+ret=$(enable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is work through wan interface after ping rule enabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+ echo "* Ping is enabled"
+else
+ echo "* Test failed!"
+ exit 1
+fi
+
+
+echo "Test Completed!"