Adding SDWAN testing for KUD containerized installer script

Please refer ICN SDWAN Module Design for architecture
link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design

Issue-ID: MULTICLOUD-956
Co-authored-by: Huifeng Le  <huifeng.le@intel.com>
Signed-off-by: r.kuralamudhan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I4bc35bc62f6bab52a5d290829f7406424d72d5ae

13 files changed, 474 insertions, 0 deletions

diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index 713847ea..f1b95acb 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -149,6 +149,21 @@ function install_plugin {
     fi
 }
 
+# install_controllers() - Install ONAP Multicloud Kubernetes controllers
+function install_controllers {
+    echo "Installing multicloud/k8s onap4k8s controllers"
+    if [[ "${testing_enabled}" == "true" ]]; then
+        echo "Test controllers installation"
+        for controller_test in sdwan; do
+            pushd $kud_tests/$controller_test
+            ansible-playbook $verbose -i \
+                $kud_inventory ${controller_test}.yml | \
+                tee $cluster_log/test-${controller_test}.log
+            popd
+        done
+    fi
+}
+
 # _print_kubernetes_info() - Prints the login Kubernetes information
 function _print_kubernetes_info {
     if ! $(kubectl version &>/dev/null); then
@@ -209,6 +224,8 @@ function install_cluster {
     if ${KUD_PLUGIN_ENABLED:-false}; then
         install_plugin
         echo "installed the install_plugin"
+        install_controllers
+        echo "installed controllers"
     fi
     _print_kubernetes_info
 }
diff --git a/kud/tests/sdwan.sh b/kud/tests/sdwan.sh
new file mode 100755
index 00000000..64b10f22
--- /dev/null
+++ b/kud/tests/sdwan.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "Create pods ..."
+kubectl apply -f sdwan/ovn-pod.yml
+kubectl apply -f sdwan/sdwan-openwrt-ovn.yml
+
+bash sdwan/test.sh
+
+echo "Clear pods ..."
+kubectl delete -f sdwan/ovn-pod.yml
+kubectl delete -f sdwan/sdwan-openwrt-ovn.yml
+
+echo "Test Completed!"
diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl
new file mode 100644
index 00000000..85c7d358
--- /dev/null
+++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl
@@ -0,0 +1,26 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+ENV http_proxy={docker_proxy}
+ENV https_proxy={docker_proxy}
+ENV no_proxy=localhost,120.0.0.1,192.168.*
+
+RUN mkdir /var/lock && \
+    opkg update && \
+    opkg install uhttpd-mod-lua && \
+    uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+    uci commit uhttpd && \
+    opkg install mwan3 && \
+    opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+ENV http_proxy=
+ENV https_proxy=
+ENV no_proxy=
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
new file mode 100644
index 00000000..8b5c57d2
--- /dev/null
+++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl
@@ -0,0 +1,19 @@
+FROM openwrt-1806-4-base
+
+#EXPOSE 80
+
+RUN mkdir /var/lock && \
+    opkg update && \
+    opkg install uhttpd-mod-lua && \
+    uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \
+    uci commit uhttpd && \
+    opkg install mwan3 && \
+    opkg install luci-app-mwan3; exit 0
+
+COPY system /etc/config/system
+COPY commands.lua /usr/lib/lua/luci/controller/
+
+USER root
+
+# using exec format so that /sbin/init is proc 1 (see procd docs)
+CMD ["/sbin/init"]
diff --git a/kud/tests/sdwan/build/README.md b/kud/tests/sdwan/build/README.md
new file mode 100644
index 00000000..87e21956
--- /dev/null
+++ b/kud/tests/sdwan/build/README.md
@@ -0,0 +1,10 @@
+# Introduction:
+Please refer ICN SDWAN Module Design for architecture introduction
+link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design
+
+# SDWAN Docker Image build instructions:
+Use below steps to build openwrt docker image: openwrt-1806-mwan3
+(1) update set_proxy file with proxy used for docker build
+(2) execute build_image.sh
+cd build
+sudo bash build_image.sh
diff --git a/kud/tests/sdwan/build/build_image.sh b/kud/tests/sdwan/build/build_image.sh
new file mode 100644
index 00000000..7ff6e20b
--- /dev/null
+++ b/kud/tests/sdwan/build/build_image.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# usage: build_images.sh
+
+set -ex
+base_image_tag=openwrt-1806-4-base
+docker_file=Dockerfile_1806_mwan3
+image_tag=openwrt-1806-mwan3
+package=openwrt-18.06.4-x86-64-generic-rootfs
+
+# build openwrt base docker images
+base_image=`docker images | grep $base_image_tag | awk '{print $1}'`
+if [ -z "$base_image" ]; then
+    # download driver source package
+    if [ ! -e /tmp/$package.tar.gz ]; then
+        wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz
+    fi
+    cp /tmp/$package.tar.gz .
+
+    docker import $package.tar.gz $base_image_tag
+fi
+
+# generate Dockerfile
+test -f ./set_proxy && . set_proxy
+docker_proxy=${docker_proxy-""}
+if [ -z "$docker_proxy" ]; then
+    cp ${docker_file}_noproxy.tpl $docker_file
+else
+    cp $docker_file.tpl $docker_file
+    sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file
+fi
+
+# build docker images for openwrt with wman3
+docker build --network=host -f $docker_file -t $image_tag .
+
+# clear
+docker image rm $base_image_tag
+rm -rf $docker_file
+rm -rf $package.tar.gz
diff --git a/kud/tests/sdwan/build/commands.lua b/kud/tests/sdwan/build/commands.lua
new file mode 100644
index 00000000..d99f4579
--- /dev/null
+++ b/kud/tests/sdwan/build/commands.lua
@@ -0,0 +1,43 @@
+-- Licensed to the public under the GNU General Public License v2.
+
+module("luci.controller.commands", package.seeall)
+
+sys = require "luci.sys"
+ut = require "luci.util"
+io = require "io"
+
+ip = "ip -4 "
+
+function index()
+    entry({"admin", "config", "command"},
+	call("execute")).dependent = false
+end
+
+function trim(s)
+    return s:match("^%s*(.-)%s*$")
+end
+
+function split_and_trim(str, sep)
+    local array = {}
+    local reg = string.format("([^%s]+)", sep)
+    for item in string.gmatch(str, reg) do
+        item_trimed = trim(item)
+        if string.len(item_trimed) > 0 then
+            table.insert(array, item_trimed)
+        end
+    end
+    return array
+end
+
+function execute()
+    local commands = luci.http.formvalue("command")
+    io.stderr:write("Execute command: %s\n" % commands)
+
+    local command_array = split_and_trim(commands, ";")
+    for index, command in ipairs(command_array) do
+        sys.exec(command)
+    end
+
+    luci.http.prepare_content("application/json")
+    luci.http.write_json("{'status':'ok'}")
+end
diff --git a/kud/tests/sdwan/build/set_proxy b/kud/tests/sdwan/build/set_proxy
new file mode 100644
index 00000000..7a195fe5
--- /dev/null
+++ b/kud/tests/sdwan/build/set_proxy
@@ -0,0 +1,2 @@
+# set docker proxy with below line, the build script will use this info
+#docker_proxy=
diff --git a/kud/tests/sdwan/build/system b/kud/tests/sdwan/build/system
new file mode 100644
index 00000000..5165430f
--- /dev/null
+++ b/kud/tests/sdwan/build/system
@@ -0,0 +1,7 @@
+config system
+    option log_file '/var/log/mylog'
+    option timezone 'UTC'
+    option ttylogin '0'
+    option log_size '64'
+    option urandom_seed '0'
+EOF
diff --git a/kud/tests/sdwan/ovn-pod.yml b/kud/tests/sdwan/ovn-pod.yml
new file mode 100644
index 00000000..0715c030
--- /dev/null
+++ b/kud/tests/sdwan/ovn-pod.yml
@@ -0,0 +1,40 @@
+# Create 2 ovn4nfv network attachment definition
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-port-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.33.0/24
+    name: subnet1
+    gateway: 172.16.33.1/24
+
+---
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: Network
+metadata:
+  name: ovn-priv-net
+spec:
+  cniType : ovn4nfv
+  ipv4Subnets:
+  - subnet: 172.16.44.0/24
+    name: subnet1
+    gateway: 172.16.44.1/24
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ovn-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+    k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+       { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+  containers:
+  - name: ovn-pod
+    image: docker.io/centos/tools:latest
+    command:
+    - /sbin/init
diff --git a/kud/tests/sdwan/sdwan-openwrt-ovn.yml b/kud/tests/sdwan/sdwan-openwrt-ovn.yml
new file mode 100644
index 00000000..2accdc6c
--- /dev/null
+++ b/kud/tests/sdwan/sdwan-openwrt-ovn.yml
@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sdwan-config-ovn
+data:
+  entrypoint.sh: |
+    #!/bin/bash
+    # Always exit on errors.
+    set -e
+
+    interface0=net0
+    ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+    interface1=net1
+    ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'`
+
+    net_config=/etc/config/network
+    cat >> $net_config << EOF
+    config interface 'wan'
+        option ifname '$interface0'
+        option proto 'static'
+        option ipaddr '$ipaddr0'
+        option netmask '255.255.255.0'
+
+    config interface 'wanb'
+        option ifname '$interface1'
+        option proto 'static'
+        option ipaddr '$ipaddr1'
+        option netmask '255.255.255.0'
+    EOF
+
+    /sbin/procd &
+    /sbin/ubusd &
+    iptables -S
+    sleep 1
+    /etc/init.d/rpcd start
+    /etc/init.d/dnsmasq start
+    /etc/init.d/network start
+    /etc/init.d/odhcpd start
+    /etc/init.d/uhttpd start
+    /etc/init.d/log start
+    /etc/init.d/dropbear start
+    /etc/init.d/mwan3 restart
+
+    echo "Entering sleep... (success)"
+
+    # Sleep forever.
+    while true; do sleep 100; done
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sdwan-ovn-pod
+  annotations:
+    k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]'
+    k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"},
+       { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}'
+spec:
+  containers:
+  - name: sdwan-ovn-pod
+    image: hle2/openwrt-1806-mwan3:v0.1.0
+    ports:
+      - containerPort: 22
+      - containerPort: 80
+    command:
+    - /bin/sh
+    - /init/entrypoint.sh
+    imagePullPolicy: IfNotPresent
+    securityContext:
+      privileged: true
+    volumeMounts:
+      - name: entrypoint-sh
+        mountPath: /init
+  volumes:
+    - name: entrypoint-sh
+      configMap:
+        name: sdwan-config-ovn
+        items:
+        - key: entrypoint.sh
+          path: entrypoint.sh
diff --git a/kud/tests/sdwan/sdwan.yml b/kud/tests/sdwan/sdwan.yml
new file mode 100644
index 00000000..760d8599
--- /dev/null
+++ b/kud/tests/sdwan/sdwan.yml
@@ -0,0 +1,44 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: create ovn network and client workload
+      command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/ovn-pod.yml"
+
+    - name: create sdwan controller
+      command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml"
+
+- hosts: kube-master
+  become: yes
+  tasks:
+    - name: install wget package for ubuntu systems
+      apt: name=wget state=present update_cache=yes
+      when: ansible_distribution == "Ubuntu"
+
+    - name: install wget package for centos systems
+      yum: name=wget state=present update_cache=yes
+      when: ansible_distribution == "CentOS"
+
+    - name: Execute sdwan test script in cluster master
+      script: test.sh
+      register: sdwan
+
+    - debug:
+        var: sdwan.stdout_lines
+
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: delete ovn network and client workload
+      command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/ovn-pod.yml"
+
+    - name: delete sdwan controller
+      command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml"
diff --git a/kud/tests/sdwan/test.sh b/kud/tests/sdwan/test.sh
new file mode 100755
index 00000000..ba4b4173
--- /dev/null
+++ b/kud/tests/sdwan/test.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+sdwan_pod_name=sdwan-ovn-pod
+ovn_pod_name=ovn-pod
+wan_interface=net0
+
+function login {
+    login_url=http://$1/cgi-bin/luci/
+    echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/')
+}
+
+function disable_ping {
+    command_url=http://$2/cgi-bin/luci/admin/config/command
+    command="uci set firewall.@rule[1].target='REJECT';fw3 reload"
+    echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function enable_ping {
+    command_url=http://$2/cgi-bin/luci/admin/config/command
+    command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload"
+    echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1)
+}
+
+function wait_for_pod {
+    status_phase=""
+    while [[ "$status_phase" != "Running" ]]; do
+        new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')"
+        if [[ "$new_phase" != "$status_phase" ]]; then
+            status_phase="$new_phase"
+        fi
+        if [[ "$new_phase" == "Err"* ]]; then
+            exit 1
+        fi
+        sleep 2
+    done
+}
+
+function wait_for_pod_namespace {
+    status_phase=""
+    while [[ "$status_phase" != "Running" ]]; do
+        new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')"
+        if [[ "$new_phase" != "$status_phase" ]]; then
+            status_phase="$new_phase"
+        fi
+        if [[ "$new_phase" == "Err"* ]]; then
+            exit 1
+        fi
+        sleep 2
+    done
+}
+
+echo "Waiting for pods to be ready ..."
+wait_for_pod $ovn_pod_name
+wait_for_pod $sdwan_pod_name
+echo "* Create pods success"
+
+sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}')
+ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}')
+echo "SDWAN pod ip:"$sdwan_pod_ip
+echo "OVN pod ip:"$ovn_pod_ip
+
+echo "Login to sdwan ..."
+security_token=""
+while [[ "$security_token" == "" ]]; do
+    echo "Get Security Token ..."
+    security_token=$(login $sdwan_pod_ip)
+    sleep 2
+done
+echo "* Security Token: "$security_token
+
+kubectl exec $sdwan_pod_name ifconfig
+
+sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface  | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}')
+echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+    echo "* Ping is work through wan interface"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+echo "Disable ping rule of wan interface ..."
+ret=$(disable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is not work through wan interface after ping rule disabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true)
+if [[ $ping_result == *", 100% packet loss"* ]]; then
+    echo "* Ping is disabled"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+echo "Enable ping rule of wan interface ..."
+ret=$(enable_ping $security_token $sdwan_pod_ip)
+
+echo "Verify ping is work through wan interface after ping rule enabled"
+ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip)
+if [[ $ping_result == *", 0% packet loss"* ]]; then
+    echo "* Ping is enabled"
+else
+    echo "* Test failed!"
+    exit 1
+fi
+
+
+echo "Test Completed!"