diff options
Diffstat (limited to 'kud')
32 files changed, 1131 insertions, 90 deletions
diff --git a/kud/build/Dockerfile b/kud/build/Dockerfile index da100bb7..38c63295 100644 --- a/kud/build/Dockerfile +++ b/kud/build/Dockerfile @@ -1,4 +1,8 @@ FROM ubuntu:18.04 as base +ARG KUD_ENABLE_TESTS=false +ARG KUD_PLUGIN_ENABLED=false +ENV KUD_ENABLE_TESTS=$KUD_ENABLE_TESTS +ENV KUD_PLUGIN_ENABLED=$KUD_PLUGIN_ENABLED ADD . /usr/src/multicloud-k8s USER root SHELL ["/bin/bash", "-c"] diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml index 1edbc6c3..72f33869 100644 --- a/kud/deployment_infra/images/sriov-daemonset.yml +++ b/kud/deployment_infra/images/sriov-daemonset.yml @@ -13,8 +13,8 @@ data: "resourceList": [{ "resourceName": "intel_sriov_700", "selectors": { - "vendors": ["8086"] - "devices": ["37cd"] + "vendors": ["8086"], + "drivers": ["i40evf", "iavf"] } }] } diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s.yml b/kud/deployment_infra/playbooks/configure-onap4k8s.yml index cacb41c9..11729171 100644 --- a/kud/deployment_infra/playbooks/configure-onap4k8s.yml +++ b/kud/deployment_infra/playbooks/configure-onap4k8s.yml @@ -19,6 +19,14 @@ repo: 'https://github.com/onap/multicloud-k8s.git' dest: /opt/multicloud + - name: install make package for ubuntu systems + apt: name=make state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install make package for centos systems + yum: name=make state=present update_cache=yes + when: ansible_distribution == "CentOS" + - name: Change the onap4k8s directory and run the command make repo command: /usr/bin/make repo register: make_repo diff --git a/kud/deployment_infra/playbooks/configure-ovn.yml b/kud/deployment_infra/playbooks/configure-ovn.yml index 3fd2c765..28de6e94 100644 --- a/kud/deployment_infra/playbooks/configure-ovn.yml +++ b/kud/deployment_infra/playbooks/configure-ovn.yml @@ -15,14 +15,6 @@ file: "{{ item }}" with_items: - "{{ ansible_os_family }}.yml" - - name: get Wand GPI files - get_url: - url: https://packages.wand.net.nz/keyring.gpg - dest: /etc/apt/trusted.gpg.d/wand.gpg - - name: add WAND Debian Repo - apt_repository: - repo: "deb https://packages.wand.net.nz {{ ansible_lsb.codename }} main" - state: present - name: install OpenVSwitch packages package: name: "{{ item }}" diff --git a/kud/deployment_infra/playbooks/configure-sriov.yml b/kud/deployment_infra/playbooks/configure-sriov.yml index 8ba6cf48..45f276c6 100644 --- a/kud/deployment_infra/playbooks/configure-sriov.yml +++ b/kud/deployment_infra/playbooks/configure-sriov.yml @@ -12,25 +12,18 @@ - hosts: localhost become: yes - pre_tasks: - - block: - - name: "End play if SRIOV is False" - debug: - msg: "SRIOV option not available, ending play" - - meta: end_play - when: SRIOV_NODE == "False" tasks: - debug: var: SRIOV_NODE - name: Apply Multus shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/multus-daemonset.yml" - when: SRIOV_NODE==True + when: SRIOV_NODE - name: Apply SRIOV CNI - shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yaml" - when: SRIOV_NODE==True + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-cni.yml" + when: SRIOV_NODE - name: Apply SRIOV DaemonSet - shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yaml" - when: SRIOV_NODE==True + shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/../images/sriov-daemonset.yml" + when: SRIOV_NODE - name: Apply SRIOV Network Attachment definition shell: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sriov-nad.yml" - when: SRIOV_NODE==True + when: SRIOV_NODE diff --git a/kud/deployment_infra/playbooks/install_iavf_drivers.sh b/kud/deployment_infra/playbooks/install_iavf_drivers.sh index d44483de..7a54e9f2 100755 --- a/kud/deployment_infra/playbooks/install_iavf_drivers.sh +++ b/kud/deployment_infra/playbooks/install_iavf_drivers.sh @@ -3,6 +3,10 @@ # Based on: # https://gerrit.akraino.org/r/#/c/icn/+/1359/1/deploy/kud-plugin-addons/device-plugins/sriov/driver/install_iavf_drivers.sh +nic_models=(XL710 X722) +nic_drivers=(i40e) +device_checkers=(is_not_used is_driver_match is_model_match) + function install_iavf_driver { local ifname=$1 @@ -27,22 +31,55 @@ function install_iavf_driver { echo '8' > /sys/class/net/$ifname/device/sriov_numvfs } -function is_used { +function is_not_used { local ifname=$1 route_info=`ip route show | grep $ifname` if [ -z "$route_info" ]; then - return 0 - else return 1 + else + return 0 + fi +} + +function is_driver_match { + local ifname=$1 + driver=`cat /sys/class/net/$ifname/device/uevent | grep DRIVER | cut -f2 -d "="` + if [ ! -z "$driver" ]; then + for nic_driver in ${nic_drivers[@]}; do + if [ "$driver" = "$nic_driver" ]; then + return 1 + fi + done + fi + return 0 +} + +function is_model_match { + local ifname=$1 + pci_addr=`cat /sys/class/net/$ifname/device/uevent | grep PCI_SLOT_NAME | cut -f2 -d "=" | cut -f2,3 -d ":"` + if [ ! -z "$pci_addr" ]; then + for nic_model in ${nic_models[@]}; do + model_match=$(lspci | grep $pci_addr | grep $nic_model) + if [ ! -z "$model_match" ]; then + return 1 + fi + done fi + return 0 } function get_sriov_ifname { for net_device in /sys/class/net/*/ ; do if [ -e $net_device/device/sriov_numvfs ] ; then ifname=$(basename $net_device) - is_used $ifname - if [ "$?" = "0" ]; then + for device_checker in ${device_checkers[@]}; do + eval $device_checker $ifname + if [ "$?" = "0" ]; then + ifname="" + break + fi + done + if [ ! -z "$ifname" ]; then echo $ifname return fi diff --git a/kud/deployment_infra/playbooks/preconfigure-sriov.yml b/kud/deployment_infra/playbooks/preconfigure-sriov.yml index c4276e1b..fd16d935 100644 --- a/kud/deployment_infra/playbooks/preconfigure-sriov.yml +++ b/kud/deployment_infra/playbooks/preconfigure-sriov.yml @@ -31,7 +31,7 @@ command: sriov/sriov_hardware_check.sh register: output - set_fact: - SRIOV: "{{ output.stdout }}" + _SRIOV: "{{ output.stdout }}" - name: Recreate the conf file for every host file: path: /tmp/sriov.conf @@ -40,7 +40,7 @@ - lineinfile : > dest=/tmp/sriov.conf create=yes - line='{{SRIOV}}' + line='{{_SRIOV}}' delegate_to: localhost - name: Clean the script and folder. file: @@ -58,32 +58,30 @@ become: yes - set_fact: SRIOV_NODE: "{{ installer_output.stdout }}" - - meta: end_play - when: SRIOV_NODE == "False" - name: Load kud variables include_vars: file: kud-vars.yml - when: SRIOV_NODE == "True" + when: SRIOV_NODE tasks: - name: Create sriov folder file: state: directory path: "{{ sriov_dest }}" - when: SRIOV_NODE == "True" ignore_errors: yes + when: SRIOV_NODE - name: Get SRIOV compatible driver get_url: "url={{ driver_url }} dest=/tmp/{{ package }}.tar.gz" - when: SRIOV_NODE == "True" + when: SRIOV_NODE - name: Extract sriov source code unarchive: src: "/tmp/{{ package }}.tar.gz" dest: "{{ sriov_dest }}" - when: SRIOV_NODE == "True" + when: SRIOV_NODE - name: Build the default target make: chdir: "/tmp/sriov/{{ package }}/src" become: yes - when: SRIOV_NODE == "True" + when: SRIOV_NODE # Copy all the driver and install script into target node - hosts: kube-node become: yes @@ -91,7 +89,7 @@ - name: Load kud variables include_vars: file: kud-vars.yml - when: SRIOV == "True" + when: _SRIOV tasks: - name: create SRIOV driver folder in the target destination file: @@ -99,18 +97,22 @@ path: "{{ item }}" with_items: - sriov_driver - when: SRIOV == "True" - - name: Copy SRIOV driver to target destination - command: "cp {{ sriov_dest }}/{{ package }}/src/iavf.ko /root/sriov_driver/" - when: SRIOV == "True" - - name: Copy SRIOV driver install script to target folder - command: "cp {{ playbook_dir }}/install_iavf_drivers.sh /root/sriov_driver/install.sh" - when: SRIOV == "True" + when: _SRIOV + - copy: + src: "{{ sriov_dest }}/{{ package }}/src/iavf.ko" + dest: sriov_driver + remote_src: no + when: _SRIOV + - copy: + src: "{{ playbook_dir }}/install_iavf_drivers.sh" + dest: sriov_driver/install.sh + remote_src: no + when: _SRIOV - name: Changing perm of "install.sh", adding "+x" - file: dest=/root/sriov_driver/install.sh mode=a+x - when: SRIOV == "True" + file: dest=sriov_driver/install.sh mode=a+x + when: _SRIOV - name: Run a script with arguments shell: ./install.sh args: - chdir: "/root/sriov_driver" - when: SRIOV == "True" + chdir: "sriov_driver" + when: _SRIOV diff --git a/kud/deployment_infra/playbooks/sriov_hardware_check.sh b/kud/deployment_infra/playbooks/sriov_hardware_check.sh index ea1b7b0c..662c28c8 100644 --- a/kud/deployment_infra/playbooks/sriov_hardware_check.sh +++ b/kud/deployment_infra/playbooks/sriov_hardware_check.sh @@ -12,14 +12,14 @@ set -o pipefail source /etc/environment -ethernet_adpator_version=$( lspci | grep "Ethernet Controller X710" | head -n 1 | cut -d " " -f 8 ) +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) if [ -z "$ethernet_adpator_version" ]; then echo "False" exit 0 fi SRIOV_ENABLED=${ethernet_adpator_version:-"false"} #checking for the right hardware version of NIC on the machine -if [ "$ethernet_adpator_version" == "X710" ]; then +if [ "$ethernet_adpator_version" == "XL710" ]; then echo "True" else echo "False" diff --git a/kud/hosting_providers/containerized/README.md b/kud/hosting_providers/containerized/README.md index 4119ca78..12ce1a19 100644 --- a/kud/hosting_providers/containerized/README.md +++ b/kud/hosting_providers/containerized/README.md @@ -27,7 +27,7 @@ Kubernetes jobs(a cluster per job) are used to install multiple clusters and log ## Quickstart Installation Guide -Build the kud docker images as follows: +Build the kud docker images as follows, add KUD_ENABLE_TESTS & KUD_PLUGIN_ENABLED for the testing only: ``` $ git clone https://github.com/onap/multicloud-k8s.git && cd multicloud-k8s @@ -38,6 +38,8 @@ $ docker build --rm \ --build-arg HTTPS_PROXY=${HTTPS_PROXY} \ --build-arg no_proxy=${no_proxy} \ --build-arg NO_PROXY=${NO_PROXY} \ + --build-arg KUD_ENABLE_TESTS=true \ + --build-arg KUD_PLUGIN_ENABLED=true \ -t github.com/onap/multicloud-k8s:latest . -f build/Dockerfile ``` Let's create a cluster-101 and cluster-102 hosts.ini as follows @@ -100,7 +102,7 @@ spec: - name: secret-volume mountPath: "/.ssh" command: ["/bin/sh","-c"] - args: ["cp -r /.ssh /root/; chmod -R 600 /root/.ssh; ./installer --cluster $CLUSTER_NAME"] + args: ["cp -r /.ssh /root/; chmod -R 600 /root/.ssh; ./installer --cluster $CLUSTER_NAME --plugins onap4k8s"] securityContext: privileged: true volumes: diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh index 52fe6279..8739ca23 100755 --- a/kud/hosting_providers/containerized/installer.sh +++ b/kud/hosting_providers/containerized/installer.sh @@ -17,10 +17,13 @@ INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" function install_prerequisites { #install package for docker images + echo "Removing ppa for jonathonf/python-3.6" + ls /etc/apt/sources.list.d/ || true + find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true apt-get update apt-get install -y curl vim wget git \ - software-properties-common python-pip - add-apt-repository ppa:longsleep/golang-backports + software-properties-common python-pip sudo + add-apt-repository -y ppa:longsleep/golang-backports apt-get update apt-get install -y golang-go rsync } @@ -100,7 +103,14 @@ function install_k8s { # install_addons() - Install Kubenertes AddOns function install_addons { - local plugins_name=$1 + if [ ${1:+1} ]; then + local plugins_name="$1" + echo "additional addons plugins $1" + else + local plugins_name="" + echo "no additional addons pluigns" + fi + source /etc/environment echo "Installing Kubernetes AddOns" ansible-galaxy install $verbose -r \ @@ -109,36 +119,52 @@ function install_addons { ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-kud.yml | \ tee $cluster_log/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd $plugins_name}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i \ $kud_inventory $kud_playbooks/configure-${addon}.yml | \ tee $cluster_log/setup-${addon}.log - if [[ "${testing_enabled}" == "true" ]]; then + done + + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov $plugins_name}; do pushd $kud_tests bash ${addon}.sh popd - fi - done + done + fi + echo "Add-ons deployment complete..." } # install_plugin() - Install ONAP Multicloud Kubernetes plugin function install_plugin { - echo "Installing multicloud/k8s plugin" - mkdir -p /opt/{kubeconfig,consul/config} - cp $HOME/.kube/config /opt/kubeconfig/kud - - pushd $kud_folder/../../../deployments - ./build.sh + echo "Installing multicloud/k8s onap4k8s plugin" if [[ "${testing_enabled}" == "true" ]]; then - ./start.sh pushd $kud_tests - for functional_test in plugin plugin_edgex plugin_fw; do - bash ${functional_test}.sh + echo "Test the onap4k8s installation" + bash onap4k8s.sh + echo "Test the onap4k8s plugin installation" + for functional_test in plugin_edgex plugin_fw; do + bash ${functional_test}.sh --external done popd fi - popd +} + +# install_controllers() - Install ONAP Multicloud Kubernetes controllers +function install_controllers { + echo "Installing multicloud/k8s onap4k8s controllers" + if [[ "${testing_enabled}" == "true" ]]; then + echo "Test controllers installation" + for controller_test in sdwan; do + pushd $kud_tests/$controller_test + ansible-playbook $verbose -i \ + $kud_inventory ${controller_test}.yml | \ + tee $cluster_log/test-${controller_test}.log + popd + done + fi } # _print_kubernetes_info() - Prints the login Kubernetes information @@ -179,6 +205,7 @@ k8s_info_file=$kud_folder/k8s_info.log testing_enabled=${KUD_ENABLE_TESTS:-false} mkdir -p /opt/csar +export CSAR_DIR=/opt/csar function install_pkg { # Install dependencies @@ -189,11 +216,19 @@ function install_pkg { function install_cluster { install_k8s $1 - install_addons $2 + if [ ${2:+1} ]; then + echo "install default addons and $2" + install_addons "$2" + else + install_addons + fi + echo "installed the addons" if ${KUD_PLUGIN_ENABLED:-false}; then install_plugin echo "installed the install_plugin" + install_controllers + echo "installed controllers" fi _print_kubernetes_info } @@ -254,7 +289,7 @@ if [ "$1" == "--cluster" ]; then cp $kud_multi_cluster_path/$cluster_name/hosts.ini $kud_inventory_folder/ cp -rf $kud_folder/inventory/group_vars $kud_inventory_folder/ - if [ -n "$3" ]; then + if [ ${3:+1} ]; then if [ "$3" == "--plugins" ]; then if [ -z "${4-}" ]; then echo "Error: plugins arguments is null; Refer the usage" @@ -262,7 +297,7 @@ if [ "$1" == "--cluster" ]; then exit 1 fi plugins_name=${@:4:$#} - install_cluster $cluster_name $plugins_name + install_cluster $cluster_name "$plugins_name" exit 0 else echo "Error: cluster argument should have plugins; \ diff --git a/kud/hosting_providers/vagrant/Vagrantfile b/kud/hosting_providers/vagrant/Vagrantfile index 58251fe9..2d1b5ab4 100644 --- a/kud/hosting_providers/vagrant/Vagrantfile +++ b/kud/hosting_providers/vagrant/Vagrantfile @@ -10,8 +10,8 @@ ############################################################################## box = { - :virtualbox => { :name => 'elastic/ubuntu-18.04-x86_64', :version => '20191013.0.0' }, - :libvirt => { :name => 'peru/ubuntu-18.04-server-amd64'} + :virtualbox => { :name => 'elastic/ubuntu-16.04-x86_64', :version => '20180708.0.0' }, + :libvirt => { :name => 'elastic/ubuntu-16.04-x86_64', :version=> '20180210.0.0'} } require 'yaml' diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index e5138c24..15974863 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -154,23 +154,19 @@ function install_addons { echo "Installing Kubernetes AddOns" _install_ansible sudo ansible-galaxy install $verbose -r $kud_infra_folder/galaxy-requirements.yml --ignore-errors - ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-kud.yml | sudo tee $log_folder/setup-kud.log - for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd}; do + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov}; do echo "Deploying $addon using configure-$addon.yml playbook.." ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log - if [[ "${testing_enabled}" == "true" ]]; then - pushd $kud_tests - bash ${addon}.sh - popd - fi done - ansible-playbook $verbose -i $kud_inventory $kud_playbooks/configure-sriov.yml | sudo tee $log_folder/setup-sriov.log - if [[ "${testing_enabled}" == "true" ]]; then + echo "Run the test cases if testing_enabled is set to true." + if [[ "${testing_enabled}" == "true" ]]; then + for addon in ${KUD_ADDONS:-virtlet ovn4nfv nfd sriov}; do pushd $kud_tests - bash sriov.sh + bash ${addon}.sh popd - fi + done + fi echo "Add-ons deployment complete..." } @@ -251,6 +247,9 @@ if [ -f $kud_folder/sources.list ]; then sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup sudo cp $kud_folder/sources.list /etc/apt/sources.list fi +echo "Removing ppa for jonathonf/python-3.6" +sudo ls /etc/apt/sources.list.d/ || true +sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true sudo apt-get update install_k8s _set_environment_file diff --git a/kud/tests/plugin_edgex.sh b/kud/tests/plugin_edgex.sh index 8eae5692..ae390add 100755 --- a/kud/tests/plugin_edgex.sh +++ b/kud/tests/plugin_edgex.sh @@ -17,7 +17,16 @@ source _common_test.sh source _functions.sh source _common.sh -base_url="http://localhost:9015/v1" +if [ ${1:+1} ]; then + if [ "$1" == "--external" ]; then + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}' | awk -F "//" '{print $2}') + onap_svc_node_port=30498 + base_url="http://$master_ip:$onap_svc_node_port/v1" + fi +fi + +base_url=${base_url:-"http://localhost:9015/v1"} kubeconfig_path="$HOME/.kube/config" csar_id=cb009bfe-bbee-11e8-9766-525400435678 rb_name="edgex" @@ -91,6 +100,9 @@ response="$(call_api -d "${payload}" "${base_url}/instance")" echo "$response" vnf_id="$(jq -r '.id' <<< "${response}")" +print_msg "Waiting for EdgeX instances" +sleep 240 + print_msg "Validating Kubernetes" kubectl get --no-headers=true --namespace=${namespace} deployment edgex-core-command kubectl get --no-headers=true --namespace=${namespace} service edgex-core-command diff --git a/kud/tests/plugin_fw.sh b/kud/tests/plugin_fw.sh index d7bed4fd..eec467c3 100755 --- a/kud/tests/plugin_fw.sh +++ b/kud/tests/plugin_fw.sh @@ -17,7 +17,16 @@ source _common_test.sh source _functions.sh source _common.sh -base_url="http://localhost:9015/v1" +if [ ${1:+1} ]; then + if [ "$1" == "--external" ]; then + master_ip=$(kubectl cluster-info | grep "Kubernetes master" | \ + awk -F ":" '{print $2}' | awk -F "//" '{print $2}') + onap_svc_node_port=30498 + base_url="http://$master_ip:$onap_svc_node_port/v1" + fi +fi + +base_url=${base_url:-"http://localhost:9015/v1"} kubeconfig_path="$HOME/.kube/config" csar_id=cc009bfe-bbee-11e8-9766-525400435678 rb_name="vfw" @@ -98,6 +107,9 @@ wait_for_pod -n "${namespace}" -l app=firewall wait_for_pod -n "${namespace}" -l app=packetgen # TODO: Provide some health check to verify vFW work +print_msg "Waiting for VNF instances" +sleep 480 + print_msg "Retrieving VNF details" call_api "${base_url}/instance/${vnf_id}" diff --git a/kud/tests/sdwan.sh b/kud/tests/sdwan.sh new file mode 100755 index 00000000..64b10f22 --- /dev/null +++ b/kud/tests/sdwan.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +echo "Create pods ..." +kubectl apply -f sdwan/ovn-pod.yml +kubectl apply -f sdwan/sdwan-openwrt-ovn.yml + +bash sdwan/test.sh + +echo "Clear pods ..." +kubectl delete -f sdwan/ovn-pod.yml +kubectl delete -f sdwan/sdwan-openwrt-ovn.yml + +echo "Test Completed!" diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl new file mode 100644 index 00000000..85c7d358 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3.tpl @@ -0,0 +1,26 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 +ENV http_proxy={docker_proxy} +ENV https_proxy={docker_proxy} +ENV no_proxy=localhost,120.0.0.1,192.168.* + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +ENV http_proxy= +ENV https_proxy= +ENV no_proxy= + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl new file mode 100644 index 00000000..8b5c57d2 --- /dev/null +++ b/kud/tests/sdwan/build/Dockerfile_1806_mwan3_noproxy.tpl @@ -0,0 +1,19 @@ +FROM openwrt-1806-4-base + +#EXPOSE 80 + +RUN mkdir /var/lock && \ + opkg update && \ + opkg install uhttpd-mod-lua && \ + uci set uhttpd.main.interpreter='.lua=/usr/bin/lua' && \ + uci commit uhttpd && \ + opkg install mwan3 && \ + opkg install luci-app-mwan3; exit 0 + +COPY system /etc/config/system +COPY commands.lua /usr/lib/lua/luci/controller/ + +USER root + +# using exec format so that /sbin/init is proc 1 (see procd docs) +CMD ["/sbin/init"] diff --git a/kud/tests/sdwan/build/README.md b/kud/tests/sdwan/build/README.md new file mode 100644 index 00000000..87e21956 --- /dev/null +++ b/kud/tests/sdwan/build/README.md @@ -0,0 +1,10 @@ +# Introduction: +Please refer ICN SDWAN Module Design for architecture introduction +link:https://wiki.akraino.org/display/AK/SDWAN+Module+Design + +# SDWAN Docker Image build instructions: +Use below steps to build openwrt docker image: openwrt-1806-mwan3 +(1) update set_proxy file with proxy used for docker build +(2) execute build_image.sh +cd build +sudo bash build_image.sh diff --git a/kud/tests/sdwan/build/build_image.sh b/kud/tests/sdwan/build/build_image.sh new file mode 100644 index 00000000..7ff6e20b --- /dev/null +++ b/kud/tests/sdwan/build/build_image.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# usage: build_images.sh + +set -ex +base_image_tag=openwrt-1806-4-base +docker_file=Dockerfile_1806_mwan3 +image_tag=openwrt-1806-mwan3 +package=openwrt-18.06.4-x86-64-generic-rootfs + +# build openwrt base docker images +base_image=`docker images | grep $base_image_tag | awk '{print $1}'` +if [ -z "$base_image" ]; then + # download driver source package + if [ ! -e /tmp/$package.tar.gz ]; then + wget -P /tmp https://downloads.openwrt.org/releases/18.06.4/targets/x86/64/$package.tar.gz + fi + cp /tmp/$package.tar.gz . + + docker import $package.tar.gz $base_image_tag +fi + +# generate Dockerfile +test -f ./set_proxy && . set_proxy +docker_proxy=${docker_proxy-""} +if [ -z "$docker_proxy" ]; then + cp ${docker_file}_noproxy.tpl $docker_file +else + cp $docker_file.tpl $docker_file + sed -i "s,{docker_proxy},$docker_proxy,g" $docker_file +fi + +# build docker images for openwrt with wman3 +docker build --network=host -f $docker_file -t $image_tag . + +# clear +docker image rm $base_image_tag +rm -rf $docker_file +rm -rf $package.tar.gz diff --git a/kud/tests/sdwan/build/commands.lua b/kud/tests/sdwan/build/commands.lua new file mode 100644 index 00000000..d99f4579 --- /dev/null +++ b/kud/tests/sdwan/build/commands.lua @@ -0,0 +1,43 @@ +-- Licensed to the public under the GNU General Public License v2. + +module("luci.controller.commands", package.seeall) + +sys = require "luci.sys" +ut = require "luci.util" +io = require "io" + +ip = "ip -4 " + +function index() + entry({"admin", "config", "command"}, + call("execute")).dependent = false +end + +function trim(s) + return s:match("^%s*(.-)%s*$") +end + +function split_and_trim(str, sep) + local array = {} + local reg = string.format("([^%s]+)", sep) + for item in string.gmatch(str, reg) do + item_trimed = trim(item) + if string.len(item_trimed) > 0 then + table.insert(array, item_trimed) + end + end + return array +end + +function execute() + local commands = luci.http.formvalue("command") + io.stderr:write("Execute command: %s\n" % commands) + + local command_array = split_and_trim(commands, ";") + for index, command in ipairs(command_array) do + sys.exec(command) + end + + luci.http.prepare_content("application/json") + luci.http.write_json("{'status':'ok'}") +end diff --git a/kud/tests/sdwan/build/set_proxy b/kud/tests/sdwan/build/set_proxy new file mode 100644 index 00000000..7a195fe5 --- /dev/null +++ b/kud/tests/sdwan/build/set_proxy @@ -0,0 +1,2 @@ +# set docker proxy with below line, the build script will use this info +#docker_proxy= diff --git a/kud/tests/sdwan/build/system b/kud/tests/sdwan/build/system new file mode 100644 index 00000000..5165430f --- /dev/null +++ b/kud/tests/sdwan/build/system @@ -0,0 +1,7 @@ +config system + option log_file '/var/log/mylog' + option timezone 'UTC' + option ttylogin '0' + option log_size '64' + option urandom_seed '0' +EOF diff --git a/kud/tests/sdwan/ovn-pod.yml b/kud/tests/sdwan/ovn-pod.yml new file mode 100644 index 00000000..0715c030 --- /dev/null +++ b/kud/tests/sdwan/ovn-pod.yml @@ -0,0 +1,40 @@ +# Create 2 ovn4nfv network attachment definition +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-port-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.33.0/24 + name: subnet1 + gateway: 172.16.33.1/24 + +--- +apiVersion: k8s.plugin.opnfv.org/v1alpha1 +kind: Network +metadata: + name: ovn-priv-net +spec: + cniType : ovn4nfv + ipv4Subnets: + - subnet: 172.16.44.0/24 + name: subnet1 + gateway: 172.16.44.1/24 + +--- +apiVersion: v1 +kind: Pod +metadata: + name: ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: ovn-pod + image: docker.io/centos/tools:latest + command: + - /sbin/init diff --git a/kud/tests/sdwan/sdwan-openwrt-ovn.yml b/kud/tests/sdwan/sdwan-openwrt-ovn.yml new file mode 100644 index 00000000..2accdc6c --- /dev/null +++ b/kud/tests/sdwan/sdwan-openwrt-ovn.yml @@ -0,0 +1,82 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sdwan-config-ovn +data: + entrypoint.sh: | + #!/bin/bash + # Always exit on errors. + set -e + + interface0=net0 + ipaddr0=`ifconfig $interface0 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + interface1=net1 + ipaddr1=`ifconfig $interface1 | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}'` + + net_config=/etc/config/network + cat >> $net_config << EOF + config interface 'wan' + option ifname '$interface0' + option proto 'static' + option ipaddr '$ipaddr0' + option netmask '255.255.255.0' + + config interface 'wanb' + option ifname '$interface1' + option proto 'static' + option ipaddr '$ipaddr1' + option netmask '255.255.255.0' + EOF + + /sbin/procd & + /sbin/ubusd & + iptables -S + sleep 1 + /etc/init.d/rpcd start + /etc/init.d/dnsmasq start + /etc/init.d/network start + /etc/init.d/odhcpd start + /etc/init.d/uhttpd start + /etc/init.d/log start + /etc/init.d/dropbear start + /etc/init.d/mwan3 restart + + echo "Entering sleep... (success)" + + # Sleep forever. + while true; do sleep 100; done + +--- +apiVersion: v1 +kind: Pod +metadata: + name: sdwan-ovn-pod + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + k8s.plugin.opnfv.org/nfn-network: '{ "type": "ovn4nfv", "interface": [{ "name": "ovn-port-net", "interface": "net0" , "defaultGateway": "false"}, + { "name": "ovn-priv-net", "interface": "net1" , "defaultGateway": "false"}]}' +spec: + containers: + - name: sdwan-ovn-pod + image: hle2/openwrt-1806-mwan3:v0.1.0 + ports: + - containerPort: 22 + - containerPort: 80 + command: + - /bin/sh + - /init/entrypoint.sh + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - name: entrypoint-sh + mountPath: /init + volumes: + - name: entrypoint-sh + configMap: + name: sdwan-config-ovn + items: + - key: entrypoint.sh + path: entrypoint.sh diff --git a/kud/tests/sdwan/sdwan.yml b/kud/tests/sdwan/sdwan.yml new file mode 100644 index 00000000..760d8599 --- /dev/null +++ b/kud/tests/sdwan/sdwan.yml @@ -0,0 +1,44 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +- hosts: localhost + become: yes + tasks: + - name: create ovn network and client workload + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/ovn-pod.yml" + + - name: create sdwan controller + command: "/usr/local/bin/kubectl apply -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" + +- hosts: kube-master + become: yes + tasks: + - name: install wget package for ubuntu systems + apt: name=wget state=present update_cache=yes + when: ansible_distribution == "Ubuntu" + + - name: install wget package for centos systems + yum: name=wget state=present update_cache=yes + when: ansible_distribution == "CentOS" + + - name: Execute sdwan test script in cluster master + script: test.sh + register: sdwan + + - debug: + var: sdwan.stdout_lines + +- hosts: localhost + become: yes + tasks: + - name: delete ovn network and client workload + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/ovn-pod.yml" + + - name: delete sdwan controller + command: "/usr/local/bin/kubectl delete -f {{ playbook_dir }}/sdwan-openwrt-ovn.yml" diff --git a/kud/tests/sdwan/test.sh b/kud/tests/sdwan/test.sh new file mode 100755 index 00000000..ba4b4173 --- /dev/null +++ b/kud/tests/sdwan/test.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +sdwan_pod_name=sdwan-ovn-pod +ovn_pod_name=ovn-pod +wan_interface=net0 + +function login { + login_url=http://$1/cgi-bin/luci/ + echo $(wget -S --spider --post-data "luci_username=root&luci_password=" $login_url 2>&1 | grep sysauth= | sed -r 's/.*sysauth=([^;]+);.*/\1/') +} + +function disable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='REJECT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function enable_ping { + command_url=http://$2/cgi-bin/luci/admin/config/command + command="uci set firewall.@rule[1].target='ACCEPT';fw3 reload" + echo $(wget -S --spider --header="Cookie:sysauth=$1" --post-data "command=$command" $command_url 2>&1) +} + +function wait_for_pod { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +function wait_for_pod_namespace { + status_phase="" + while [[ "$status_phase" != "Running" ]]; do + new_phase="$(kubectl get pods -o wide -n $2 | grep ^$1 | awk '{print $3}')" + if [[ "$new_phase" != "$status_phase" ]]; then + status_phase="$new_phase" + fi + if [[ "$new_phase" == "Err"* ]]; then + exit 1 + fi + sleep 2 + done +} + +echo "Waiting for pods to be ready ..." +wait_for_pod $ovn_pod_name +wait_for_pod $sdwan_pod_name +echo "* Create pods success" + +sdwan_pod_ip=$(kubectl get pods -o wide | grep ^$sdwan_pod_name | awk '{print $6}') +ovn_pod_ip=$(kubectl get pods -o wide | grep ^$ovn_pod_name | awk '{print $6}') +echo "SDWAN pod ip:"$sdwan_pod_ip +echo "OVN pod ip:"$ovn_pod_ip + +echo "Login to sdwan ..." +security_token="" +while [[ "$security_token" == "" ]]; do + echo "Get Security Token ..." + security_token=$(login $sdwan_pod_ip) + sleep 2 +done +echo "* Security Token: "$security_token + +kubectl exec $sdwan_pod_name ifconfig + +sdwan_pod_wan_ip=$(kubectl exec $sdwan_pod_name ifconfig $wan_interface | awk '/inet/{print $2}' | cut -f2 -d ":" | awk 'NR==1 {print $1}') +echo "Verify ping is work through wan interface between $sdwan_pod_name and $ovn_pod_name" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is work through wan interface" +else + echo "* Test failed!" + exit 1 +fi + +echo "Disable ping rule of wan interface ..." +ret=$(disable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is not work through wan interface after ping rule disabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip 2>&1 || true) +if [[ $ping_result == *", 100% packet loss"* ]]; then + echo "* Ping is disabled" +else + echo "* Test failed!" + exit 1 +fi + +echo "Enable ping rule of wan interface ..." +ret=$(enable_ping $security_token $sdwan_pod_ip) + +echo "Verify ping is work through wan interface after ping rule enabled" +ping_result=$(kubectl exec $ovn_pod_name -- ping -c 3 $sdwan_pod_wan_ip) +if [[ $ping_result == *", 0% packet loss"* ]]; then + echo "* Ping is enabled" +else + echo "* Test failed!" + exit 1 +fi + + +echo "Test Completed!" diff --git a/kud/tests/sriov.sh b/kud/tests/sriov.sh index c66f5db8..a721b722 100755 --- a/kud/tests/sriov.sh +++ b/kud/tests/sriov.sh @@ -10,13 +10,13 @@ set -o pipefail -ethernet_adpator_version=$( lspci | grep "Ethernet Controller X710" | head -n 1 | cut -d " " -f 8 ) +ethernet_adpator_version=$( lspci | grep "Ethernet Controller XL710" | head -n 1 | cut -d " " -f 8 ) if [ -z "$ethernet_adpator_version" ]; then echo " Ethernet adapator version is not set. SRIOV test case cannot run on this machine" exit 0 fi #checking for the right hardware version of NIC on the machine -if [ $ethernet_adpator_version == "X710" ]; then +if [ $ethernet_adpator_version == "XL710" ]; then echo "NIC card specs match. SRIOV option avaiable for this version." else echo -e "Failed. The version supplied does not match.\nTest cannot be executed." diff --git a/kud/tests/vIPSec/README.md b/kud/tests/vIPSec/README.md new file mode 100644 index 00000000..3046db7a --- /dev/null +++ b/kud/tests/vIPSec/README.md @@ -0,0 +1,36 @@ +# vIPSec use case in ONAP +This use case is composed of four virtual functions (VFs) including two +IPSec gateways, a packet generator and a traffic sink, each running in +separate Ubuntu Virtual Machines: + + * [Packet generator][1]: Sends packets to the packet sink through the +tunnel constructed thru IPSec. This includes a script that installs the +packet generator based on packetgen[4]. + * [IPsec gateways][2]: Two IPSec gateways constructed the secure tunnel +for traffic transportation. This includes a script to install and configure +the IPSec gateways thru VPP. + * [Traffic sink][3]: Displays the traffic volume that lands at the sink +VM using the link http://192.168.80.250:667 through your browser +and enable automatic page refresh by clicking the "Off" button. You +can see the traffic volume in the charts. + +This set of scripts aims to construct the vIPSec use case in order to set +up a secure tunnel between peers and improve its performance along with +hardware acceleration technologies such as SRIOV and QAT. + +User can apply the helm chart named 'vipsec' inside the k8s/kud/demo folder +to set up the whole use case. A fully-functional Kubernetes cluster, Virtlet +as well as ovn4nfv-k8s[5] plugin need to be pre-installed for the usage. +*[Place needs improvements] After having the virtual machines ready, please +manually change the MAC address inside the ipsec.conf to enable the routing. +And also start up the packetgen to send packet with src and dst defined in +the templates/values.yaml inside the helm chart. Detail instructions will be +put inside the helm chart. + +If you'd like to test the performance with QAT/SRIOV involved, first get +these hardwares pre-configured. Then change the value of 'qat_enabled' and +'sriov_enabled' inside templates/values.yaml of the helm chart accordingly. +User could observe variance in throughput inside the traffic sink. + +[4] https://pktgen-dpdk.readthedocs.io/en/latest/ +[5] https://github.com/opnfv/ovn4nfv-k8s-plugin diff --git a/kud/tests/vIPSec/ipsec b/kud/tests/vIPSec/ipsec new file mode 100755 index 00000000..4b278574 --- /dev/null +++ b/kud/tests/vIPSec/ipsec @@ -0,0 +1,163 @@ +#!/bin/bash +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2019 Intel Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# This script prepares the runtime environment +# for running vIPSec shell scripts on Ubuntu 18.04 + +set -o nounset +set -o pipefail +set -o xtrace +set -o errexit + +function setup_dependencies { + apt-get update + apt-get install -y curl gnupg2 pciutils make gcc libnuma-dev python git linux-headers-`uname -r` module-init-tools libssl-dev + echo "deb [trusted=yes] https://packagecloud.io/fdio/release/ubuntu bionic main" >> /etc/apt/sources.list.d/99fd.io.list + curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - +} + +function install_vpp { + apt-get update + apt-get install -y vpp vpp-plugin-core vpp-plugin-dpdk +} + +function install_dpdk { + cd /opt + git clone http://dpdk.org/git/dpdk + cd /opt/dpdk + export RTE_TARGET=x86_64-native-linux-gcc/ && export DESTDIR=/opt/dpdk && export RTE_SDK=/opt/dpdk && make install T=x86_64-native-linux-gcc + modprobe uio + insmod x86_64-native-linux-gcc/kmod/igb_uio.ko +} + +function ipsec_settings { +# Create vpp configuration file + cat > /opt/config/vpp.config << EOF + unix { + exec /opt/config/ipsec.conf + nodaemon + cli-listen /run/vpp/cli.sock + log /tmp/vpp.log + } + + cpu { + main-core 0 + corelist-workers 1 + } + + dpdk { + socket-mem 512 + log-level debug + no-tx-checksum-offload + dev default{ + num-tx-desc 512 + num-rx-desc 512 + } + dev interfaceABus + { + workers 0 + } + dev interfaceBBus + { + workers 0 + } + vdev crypto_aesni_mb0 + + no-multi-seg + + #enable_cryptodev + + } +EOF + +# Check if sriov and qat are enabled, bind the pci devices with igb_uio driver + if [ "$sriov_enabled" = true ]; then + export interfaceABus=$(lspci -D -nn | grep -m1 '8086:154c' | cut -d ' ' -f 1) + export interfaceBBus=$(lspci -D -nn | grep -m2 '8086:154c' | cut -d ' ' -f 1 | tail -n1) + else + export interfaceABus=$(ls -la /sys/class/net | grep 'eth1' | cut -d '/' -f 5) + export interfaceBBus=$(ls -la /sys/class/net | grep 'eth3' | cut -d '/' -f 5) + fi + sed -i -e "s/interfaceABus/${interfaceABus}/g" -e "s/interfaceBBus/${interfaceBBus}/g" /opt/config/vpp.config + python /opt/dpdk/usertools/dpdk-devbind.py -b igb_uio $interfaceABus $interfaceBBus + export interfaceA=$(vppctl sh int | awk '$2 == "1"' | cut -d ' ' -f 1) + export interfaceB=$(vppctl sh int | awk '$2 == "2"' | cut -d ' ' -f 1) + + if [ "$qat_enabled" = true ]; then + export qatABus=$(lspci -D -nn | grep -m1 '8086:37c9' | cut -d ' ' -f 1) + export qatBBus=$(lspci -D -nn | grep -m2 '8086:37c9' | cut -d ' ' -f 1 | tail -n1) + python /opt/dpdk/usertools/dpdk-devbind.py -b igb_uio $qatABus $qatBBus + sed -i "/#enable_cryptodev/a\n dev $qatABus\n dev $qatBBus\n" /opt/config/vpp.config + sed -i "/vdev crypto_aesni_mb0/d" /opt/config/vpp.config + fi + +# Create the sample ipsec configuration file + cat > /opt/config/ipsec.conf << EOF + set interface state VirtualFunctionEthernet0/5/0 up + set interface state VirtualFunctionEthernet0/6/0 up + + set interface ip address VirtualFunctionEthernet0/5/0 input_interface_ip/24 + set interface ip address VirtualFunctionEthernet0/6/0 output_interface_ip/24 + + set int promiscuous on VirtualFunctionEthernet0/5/0 + set int promiscuous on VirtualFunctionEthernet0/6/0 + + set ip arp VirtualFunctionEthernet0/6/0 remote_tunnel_ip fa:16:3e:a6:e4:c7 + set ip arp VirtualFunctionEthernet0/5/0 input_interface_ip fa:16:3e:f1:65:dc + + ip route add count 1 packet_dst/32 via route_interface VirtualFunctionEthernet0/6/0 + + ipsec spd add 1 + set interface ipsec spd VirtualFunctionEthernet0/6/0 1 + ipsec sa add 1 spi 1921681003 esp tunnel-src output_interface_ip tunnel-dst remote_tunnel_ip crypto-key 2b7e151628aed2a6abf7158809cf4f3d crypto-alg aes-cbc-128 integ-key 6867666568676665686766656867666568676669 integ-alg sha1-96 + ipsec policy add spd 1 traffic_direction priority 100 action protect sa 1 local-ip-range packet_src-packet_src remote-ip-range packet_dst-packet_dst + ipsec policy add spd 1 traffic_direction priority 90 protocol 50 action bypass local-ip-range packet_src-255.255.255.255 remote-ip-range remote_tunnel_ip-remote_tunnel_ip +EOF + +# Replace all ip and interfaces inside the ipsec configuration file + sed -i -e "s/input_interface_ip/${input_interface_ip}/g" -e "s/output_interface_ip/${output_interface_ip}/g" -e "s/remote_tunnel_ip/${remote_tunnel_ip}/g" -e "s/route_interface/${route_interface}/g" -e "s#VirtualFunctionEthernet0/5/0#${interfaceA}#g" -e "s#VirtualFunctionEthernet0/6/0#${interfaceB}/g" -e "s/packet_src/${packet_src}/g" -e "s/packet_dst/${packet_dst}/g" -e "s/traffic_direction/${traffic_direction}/g" /opt/config/ipsec.conf + vpp -c /opt/config/vpp.config +} + + +mkdir /opt/config +echo "$demo_artifacts_version" > /opt/config/demo_artifacts_version.txt +echo "$dcae_collector_ip" > /opt/config/dcae_collector_ip.txt +echo "$dcae_collector_port" > /opt/config/dcae_collector_port.txt +echo "$ipsec_private_net_gw" > /opt/config/ipsec_private_net_gw_ip.txt +echo "$ipsec_private_net_cidr" > /opt/config/ipsec_private_net_cidr.txt +echo "$ipsec_private_network_name" > /opt/config/ipsec_private_network_name.txt +echo "$packet_src" > /opt/config/packet_source_ip.txt +echo "$packet_dst" > /opt/config/packet_destination_ip.txt +echo "$remote_tunnel_ip" > /opt/config/remote_tunnel.txt +echo "$route_interface" > /opt/config/route_interface.txt +echo "$traffic_direction" > /opt/config/traffic_direction.txt +echo "$vipsecA_private_ip_0" > /opt/config/vipsecA_private_ip0.txt +echo "$vipsecA_private_ip_2" > /opt/config/vipsecA_private_ip2.txt +echo "$protected_clientA_network_name" > /opt/config/protected_clientA_network_name.txt +echo "$protected_clientA_net_gw" > /opt/config/protected_clientA_net_gw.txt +echo "$protected_clientA_net_cidr" > /opt/config/protected_clientA_net_cidr.txt + +echo 'vm.nr_hugepages = 1024' >> /etc/sysctl.conf +sysctl -p + +setup_dependencies +install_vpp +install_dpdk +ipsec_settings diff --git a/kud/tests/vIPSec/pktgen b/kud/tests/vIPSec/pktgen new file mode 100755 index 00000000..14d7e6ca --- /dev/null +++ b/kud/tests/vIPSec/pktgen @@ -0,0 +1,77 @@ +#!/bin/bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2019 Intel Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# This script prepares the runtime environment +# for running vIPSec shell scripts on Ubuntu18.04 + +set -o nounset +set -o pipefail +set -o xtrace +set -o errexit + + +DPDK_DIR=$PWD/dpdk +Pktgen_Dir=$PWD/pktgen-dpdk + +function setup_dependencies { + sudo apt-get update + git clone http://dpdk.org/git/dpdk + git clone http://dpdk.org/git/apps/pktgen-dpdk + KERNEL_VERSION=$(uname -r) + echo $KERNEL_VERSION + sudo apt-get install -y linux-headers-$KERNEL_VERSION libpcap-dev gcc make libnuma-dev liblua5.3-dev python +} + +function build_dpdk { + export RTE_SDK=$DPDK_DIR + export RTE_TARGET=x86_64-native-linux-gcc + export DESTDIR=$DPDK_DIR + cd $RTE_SDK + make install T=x86_64-native-linux-gcc + echo "DPDK install finished" + modprobe uio + insmod x86_64-native-linux-gcc/kmod/igb_uio.ko + export interface=$(lspci -nn | grep -m1 'Ethernet controller' | cut -d ' ' -f 1) + python ./usertools/dpdk-devbind.py -b igb_uio $interface +} + +function build_pktgen { + cd $Pktgen_Dir + export RTE_SDK=$DPDK_DIR + export RTE_TARGET=x86_64-native-linux-gcc + make +} + +mkdir /opt/config +echo "$demo_artifacts_version" > /opt/config/demo_artifacts_version.txt +echo "$vpg_private_ip_0" > /opt/config/vpg_private_ip0.txt +echo "$ipsec_a_private_ip_0" > /opt/config/ipsec_a_private_ip0.txt +echo "$protected_clientA_network_name" > /opt/config/protected_clientA_network_name.txt +echo "$dcae_collector_ip" > /opt/config/dcae_collector_ip.txt +echo "$dcae_collector_port" > /opt/config/dcae_collector_port.txt +echo "$protected_clientA_net_gw" > /opt/config/protected_clientA_net_gw.txt +echo "$protected_clientA_net_cidr" > /opt/config/protected_clientA_net_cidr.txt + +echo 'vm.nr_hugepages = 1024' >> /etc/sysctl.conf +sysctl -p + +setup_dependencies +build_dpdk +build_pktgen diff --git a/kud/tests/vIPSec/remote_ipsec b/kud/tests/vIPSec/remote_ipsec new file mode 100755 index 00000000..6a676c96 --- /dev/null +++ b/kud/tests/vIPSec/remote_ipsec @@ -0,0 +1,164 @@ +#!/bin/bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2019 Intel Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# This script prepares the runtime environment +# for running vIPSec shell scripts on Ubuntu18.04 + +set -o nounset +set -o pipefail +set -o xtrace +set -o errexit + +function setup_dependencies { + apt-get update + apt-get install -y curl gnupg2 pciutils make gcc libnuma-dev python git linux-headers-`uname -r` module-init-tools libssl-dev + echo "deb [trusted=yes] https://packagecloud.io/fdio/release/ubuntu bionic main" >> /etc/apt/sources.list.d/99fd.io.list + curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - +} + +function install_vpp { + apt-get update + apt-get install -y vpp vpp-plugin-core vpp-plugin-dpdk +} + +function install_dpdk { + cd /opt + git clone http://dpdk.org/git/dpdk + cd /opt/dpdk + export RTE_TARGET=x86_64-native-linux-gcc/ && export DESTDIR=/opt/dpdk && export RTE_SDK=/opt/dpdk && make install T=x86_64-native-linux-gcc + modprobe uio + insmod x86_64-native-linux-gcc/kmod/igb_uio.ko +} + +function ipsec_settings { +# Create vpp configuration file + cat > /opt/config/vpp.config << EOF + unix { + exec /opt/config/ipsec.conf + nodaemon + cli-listen /run/vpp/cli.sock + log /tmp/vpp.log + } + + cpu { + main-core 0 + corelist-workers 1 + } + + dpdk { + socket-mem 512 + log-level debug + no-tx-checksum-offload + dev default{ + num-tx-desc 512 + num-rx-desc 512 + } + dev interfaceABus + { + workers 0 + } + dev interfaceBBus + { + workers 0 + } + vdev crypto_aesni_mb0 + + no-multi-seg + + #enable_cryptodev + + } +EOF + +# Check if sriov and qat are enabled, bind the pci devices with igb_uio driver + if [ "$sriov_enabled" = true ]; then + export interfaceABus=$(lspci -D -nn | grep -m1 '8086:154c' | cut -d ' ' -f 1) + export interfaceBBus=$(lspci -D -nn | grep -m2 '8086:154c' | cut -d ' ' -f 1 | tail -n1) + else + export interfaceABus=$(ls -la /sys/class/net | grep 'eth1' | cut -d '/' -f 5) + export interfaceBBus=$(ls -la /sys/class/net | grep 'eth3' | cut -d '/' -f 5) + fi + sed -i -e "s/interfaceABus/${interfaceABus}/g" -e "s/interfaceBBus/${interfaceBBus}/g" /opt/config/vpp.config + python /opt/dpdk/usertools/dpdk-devbind.py -b igb_uio $interfaceABus $interfaceBBus + export interfaceA=$(vppctl sh int | awk '$2 == "1"' | cut -d ' ' -f 1) + export interfaceB=$(vppctl sh int | awk '$2 == "2"' | cut -d ' ' -f 1) + + if [ "$qat_enabled" = true ]; then + export qatABus=$(lspci -D -nn | grep -m1 '8086:37c9' | cut -d ' ' -f 1) + export qatBBus=$(lspci -D -nn | grep -m2 '8086:37c9' | cut -d ' ' -f 1 | tail -n1) + python /opt/dpdk/usertools/dpdk-devbind.py -b igb_uio $qatABus $qatBBus + sed -i "/#enable_cryptodev/a\n dev $qatABus\n dev $qatBBus\n" /opt/config/vpp.config + sed -i "/vdev crypto_aesni_mb0/d" /opt/config/vpp.config + fi + +# Create ipsec configuration file + cat > /opt/config/ipsec.conf << EOF + set interface state VirtualFunctionEthernet0/5/0 up + set interface state VirtualFunctionEthernet0/6/0 up + + set interface ip address VirtualFunctionEthernet0/5/0 input_interface_ip/24 + set interface ip address VirtualFunctionEthernet0/6/0 output_interface_ip/24 + + set int promiscuous on VirtualFunctionEthernet0/5/0 + set int promiscuous on VirtualFunctionEthernet0/6/0 + + set ip arp VirtualFunctionEthernet0/6/0 remote_tunnel_ip fa:16:3e:a6:e4:c7 + set ip arp VirtualFunctionEthernet0/5/0 routing_ip fa:16:3e:f1:65:dc + + ip route add count 1 packet_dst/32 via route_interface VirtualFunctionEthernet0/6/0 + + ipsec spd add 1 + set interface ipsec spd VirtualFunctionEthernet0/6/0 1 + ipsec sa add 1 spi 1921681004 esp tunnel-src local_tunnel_ip tunnel-dst remote_tunnel_ip crypto-key 2b7e151628aed2a6abf7158809cf4f3d crypto-alg aes-cbc-128 integ-key 6867666568676665686766656867666568676669 integ-alg sha1-96 + ipsec policy add spd 1 traffic_direction priority 100 action protect sa 1 local-ip-range packet_src-packet_src remote-ip-range packet_dst-packet_dst + ipsec policy add spd 1 traffic_direction priority 90 protocol 50 action bypass local-ip-range packet_src-255.255.255.255 remote-ip-range remote_tunnel_ip-remote_tunnel_ip +EOF + +# Replace the actual ip and interfaces into the ipsec configuration + sed -i -e "s/input_interface_ip/${input_interface_ip}/g" -e "s/output_interface_ip/${output_interface_ip}/g" -e "s/routing_ip/${vsn_private_ip_0}/g" -e "s#VirtualFunctionEthernet0/5/0#${interfaceA}#g" -e "s#VirtualFunctionEthernet0/6/0#${interfaceB}#g" -e "s/local_tunnel_ip/${local_tunnel_ip}/g" -e "s/remote_tunnel_ip/${remote_tunnel_ip}/g" -e "s/route_interface/${route_interface}/g" -e "s/packet_src/${packet_src}/g" -e "s/packet_dst/${packet_dst}/g" -e "s/traffic_direction/${traffic_direction}/g" /opt/config/ipsec.conf + vpp -c /opt/config/vpp.config +} + + +mkdir /opt/config +echo "$demo_artifacts_version" > /opt/config/demo_artifacts_version.txt +echo "$dcae_collector_ip" > /opt/config/dcae_collector_ip.txt +echo "$dcae_collector_port" > /opt/config/dcae_collector_port.txt +echo "$ipsec_private_net_gw" > /opt/config/ipsec_private_net_gw_ip.txt +echo "$ipsec_private_net_cidr" > /opt/config/ipsec_private_net_cidr.txt +echo "$ipsec_private_network_name" > /opt/config/ipsec_private_network_name.txt +echo "$packet_src" > /opt/config/packet_source_ip.txt +echo "$packet_dst" > /opt/config/packet_destination_ip.txt +echo "$remote_tunnel_ip" > /opt/config/remote_tunnel.txt +echo "$route_interface" > /opt/config/route_interface.txt +echo "$traffic_direction" > /opt/config/traffic_direction.txt +echo "$vipsecB_private_ip_0" > /opt/config/vipsecB_private_ip0.txt +echo "$vipsecB_private_ip_2" > /opt/config/vipsecB_private_ip2.txt +echo "$protected_clientB_network_name" > /opt/config/protected_clientB_network_name.txt +echo "$protected_clientB_net_gw" > /opt/config/protected_clientB_net_gw.txt +echo "$protected_clientB_net_cidr" > /opt/config/protected_clientB_net_cidr.txt + +echo 'vm.nr_hugepages = 1024' >> /etc/sysctl.conf +sysctl -p + +setup_dependencies +install_vpp +install_dpdk +ipsec_settings diff --git a/kud/tests/vIPSec/sink b/kud/tests/vIPSec/sink new file mode 100755 index 00000000..c180d43c --- /dev/null +++ b/kud/tests/vIPSec/sink @@ -0,0 +1,48 @@ +#!/bin/bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2019 Intel Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +# This script prepares the runtime environment +# for running vIPSec shell scripts on Ubuntu 18.04 + +set -o nounset +set -o pipefail +set -o xtrace +set -o errexit + +function setup_dependencies { + apt-get update + apt install -y wget darkstat net-tools unzip + + # Configure and run Darkstat + sed -i "s/START_DARKSTAT=.*/START_DARKSTAT=yes/g;s/INTERFACE=.*/INTERFACE=\"-i eth1\"/g" /etc/darkstat/init.cfg + + systemctl restart darkstat +} + +mkdir -p /opt/config/ +echo "$protected_net_cidr" > /opt/config/protected_net_cidr.txt +echo "$vfw_private_ip_0" > /opt/config/fw_ipaddr.txt +echo "$vsn_private_ip_0" > /opt/config/sink_ipaddr.txt +echo "$demo_artifacts_version" > /opt/config/demo_artifacts_version.txt +echo "$protected_net_gw" > /opt/config/protected_net_gw.txt +echo "$protected_private_net_cidr" > /opt/config/unprotected_net.txt + +setup_dependencies + |