From 216bd6a4870c680555d586f0010a633d5fa15700 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorek Date: Fri, 29 Nov 2019 15:15:51 +0100 Subject: Import Vagrant environment from test/security/k8s Infrastructure mockup has been previously set up for CIS guidelines checking. Empty Kubernetes cluster was sufficient for that purpose. It will be adjusted to satisfy minimal ONAP requirements and should eventually supersede previous testing environment. Issue-ID: ONAPARC-537 Change-Id: Iada29d86642b8a5513e9d1bbd895db2094ad12b9 Signed-off-by: Pawel Wieczorek --- bootstrap/vagrant-minimal-onap/Vagrantfile | 174 +++++++++++++++++++++ bootstrap/vagrant-minimal-onap/config/cluster.yml | 49 ++++++ .../tools/config/95silent-approval | 2 + .../vagrant-minimal-onap/tools/config/dot_curlrc | 8 + .../vagrant-minimal-onap/tools/config/dot_wgetrc | 2 + .../tools/get_customization_scripts.sh | 5 + .../vagrant-minimal-onap/tools/get_kubectl.sh | 41 +++++ bootstrap/vagrant-minimal-onap/tools/get_rke.sh | 39 +++++ .../tools/imported/openstack-k8s-controlnode.sh | 31 ++++ .../tools/imported/openstack-k8s-workernode.sh | 34 ++++ .../vagrant-minimal-onap/tools/setup_kubectl.sh | 52 ++++++ 11 files changed, 437 insertions(+) create mode 100644 bootstrap/vagrant-minimal-onap/Vagrantfile create mode 100644 bootstrap/vagrant-minimal-onap/config/cluster.yml create mode 100644 bootstrap/vagrant-minimal-onap/tools/config/95silent-approval create mode 100644 bootstrap/vagrant-minimal-onap/tools/config/dot_curlrc create mode 100644 bootstrap/vagrant-minimal-onap/tools/config/dot_wgetrc create mode 100755 bootstrap/vagrant-minimal-onap/tools/get_customization_scripts.sh create mode 100755 bootstrap/vagrant-minimal-onap/tools/get_kubectl.sh create mode 100755 bootstrap/vagrant-minimal-onap/tools/get_rke.sh create mode 100644 bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-controlnode.sh create mode 100644 bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-workernode.sh create mode 100755 bootstrap/vagrant-minimal-onap/tools/setup_kubectl.sh (limited to 'bootstrap/vagrant-minimal-onap') diff --git a/bootstrap/vagrant-minimal-onap/Vagrantfile b/bootstrap/vagrant-minimal-onap/Vagrantfile new file mode 100644 index 000000000..1ccc3ef9f --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/Vagrantfile @@ -0,0 +1,174 @@ +# -*- mode: ruby -*- +# -*- coding: utf-8 -*- + +host_ip = "192.168.121.1" +operator_key = "${HOME}/.ssh/onap-key" +vagrant_user = "vagrant" +vagrant_password = "vagrant" +synced_folder_main = "/vagrant" +synced_folder_config = "#{synced_folder_main}/config" +cluster_yml = "cluster.yml" +apt_prefs_dir = "/etc/apt/apt.conf.d" +apt_prefs = "95silent-approval" + +vm_memory = 2 * 1024 +vm_cpus = 1 +vm_box = "generic/ubuntu1804" + +operation = { name: 'operator', hostname: 'operator', ip: '172.17.4.254' } +cluster = [ + { name: 'control', hostname: 'control', ip: '172.17.4.100' }, + { name: 'worker', hostname: 'worker', ip: '172.17.4.101' } +] + +all = cluster.dup << operation + +operation_post_msg = "Run: \"vagrant provision #{operation[:name]} --provision-with=rke_up,setup_kubectl\" to complete cluster creation" + +$replace_dns = <<-SCRIPT + HOST_IP="$1" + rm -f /etc/resolv.conf # drop its dynamic management by systemd-resolved + echo nameserver "$HOST_IP" | tee /etc/resolv.conf +SCRIPT + +$add_to_docker_group = <<-SCRIPT + USER="$1" + echo "Adding ${USER} to 'docker' group" + usermod -aG docker "$USER" +SCRIPT + +$setup_debconf = <<-SCRIPT + echo "Setting debconf frontend to noninteractive" + sed -i'.orig' '/^Config:/a Frontend: noninteractive' /etc/debconf.conf +SCRIPT + +$install_sshpass = <<-SCRIPT + apt-get update + echo "Installing 'sshpass'" + apt-get install sshpass +SCRIPT + +$generate_key = <<-SCRIPT + KEY_FILE="$1" + echo "Generating SSH key (${KEY_FILE})" + ssh-keygen -q -b 4096 -t rsa -f "$KEY_FILE" -N "" +SCRIPT + +$deploy_key = <<-SCRIPT + KEY="$1" + USER="$2" + PASS="$PASSWORD" + IPS="$3" + echo "Deploying ${KEY} for ${USER}" + for ip in $IPS; do + echo "on ${ip}" + sshpass -p "$PASS" ssh-copy-id -o StrictHostKeyChecking=no -i "$KEY" "${USER}@${ip}" + done +SCRIPT + +$link_dotfiles = <<-SCRIPT + SYNC_DIR="$1" + for rc in ${SYNC_DIR}/dot_*; do + src="$rc" + dst="${HOME}/.${rc##*dot_}" + echo "Symlinking ${src} to ${dst}" + ln -sf "$src" "$dst" + done +SCRIPT + +$link_file = <<-SCRIPT + SYNC_DIR="$1" + FILE="$2" + src="${SYNC_DIR}/${FILE}" + dst="$3" + echo "Symlinking ${src} to ${dst}" + ln -sf "$src" "$dst" +SCRIPT + +$rke_up = "rke up" +$rke_down = "rke remove --force" + +Vagrant.configure('2') do |config| + all.each do |machine| + config.vm.define machine[:name] do |config| + config.vm.box = vm_box + config.vm.hostname = machine[:hostname] + + config.vm.provider :virtualbox do |v| + v.name = machine[:name] + v.memory = vm_memory + v.cpus = vm_cpus + end + + config.vm.provider :libvirt do |v| + v.memory = vm_memory + v.cpus = vm_cpus + end + + config.vm.network :private_network, ip: machine[:ip] + config.vm.provision "replace_dns", type: :shell, run: "always", inline: $replace_dns, args: host_ip + + if machine[:name] == 'control' + config.vm.provision "customize_control", type: :shell, path: "../../tools/dublin/imported/openstack-k8s-controlnode.sh" + config.vm.provision "fix_groups_control", type: :shell, inline: $add_to_docker_group, args: vagrant_user + end + + if machine[:name] == 'worker' + config.vm.provision "customize_worker", type: :shell, path: "../../tools/dublin/imported/openstack-k8s-workernode.sh" + config.vm.provision "fix_group_worker", type: :shell, inline: $add_to_docker_group, args: vagrant_user + end + + if machine[:name] == 'operator' + config.vm.synced_folder ".", synced_folder_main, type: "rsync", rsync__exclude: "Vagrantfile" + config.vm.synced_folder "../../tools/config", synced_folder_config, type: "rsync" + + config.vm.provision "setup_debconf", type: :shell, inline: $setup_debconf + config.vm.provision "link_apt_prefs", type: :shell, run: "always" do |s| + s.inline = $link_file + s.args = [synced_folder_config, apt_prefs, apt_prefs_dir] + end + config.vm.provision "link_dotfiles_root", type: :shell, run: "always" do |s| + s.inline = $link_dotfiles + s.args = synced_folder_config + end + config.vm.provision "link_dotfiles_user", type: :shell, run: "always" do |s| + s.privileged = false + s.inline = $link_dotfiles + s.args = synced_folder_config + end + + config.vm.provision "install_sshpass", type: :shell, inline: $install_sshpass + config.vm.provision "generate_key", type: :shell, privileged: false, inline: $generate_key, args: operator_key + + ips = "" + cluster.each { |node| ips << node[:ip] << " " } + config.vm.provision "deploy_key", type: :shell do |s| + s.privileged = false + s.inline = $deploy_key + s.args = [operator_key, vagrant_user, ips] + s.env = {'PASSWORD': vagrant_password} + end + + config.vm.provision "get_rke", type: :shell, path: "../../tools/dublin/get_rke.sh" + config.vm.provision "link_cluster_yml", type: :shell, run: "always" do |s| + s.privileged = false + s.inline = $link_file + s.args = [synced_folder_main, cluster_yml, "$HOME"] + end + + config.vm.post_up_message = operation_post_msg + config.vm.provision "rke_up", type: :shell, run: "never", privileged: false, inline: $rke_up + config.trigger.before :destroy do |trigger| + trigger.warn = "Removing cluster" + trigger.run_remote = {privileged: false, inline: $rke_down} + end + + config.vm.provision "get_kubectl", type: :shell, path: "../../tools/dublin/get_kubectl.sh" + config.vm.provision "setup_kubectl", type: :shell, run: "never" do |s| + s.privileged = false + s.path = "../../tools/dublin/setup_kubectl.sh" + end + end + end + end +end diff --git a/bootstrap/vagrant-minimal-onap/config/cluster.yml b/bootstrap/vagrant-minimal-onap/config/cluster.yml new file mode 100644 index 000000000..df93a8863 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/config/cluster.yml @@ -0,0 +1,49 @@ +# An example of a Kubernetes cluster for ONAP +ssh_key_path: &ssh_key_path "~/.ssh/onap-key" +nodes: +- address: 172.17.4.100 + port: "22" + role: + - controlplane + - etcd + hostname_override: "onap-control-1" + user: vagrant + ssh_key_path: *ssh_key_path +- address: 172.17.4.101 + port: "22" + role: + - worker + hostname_override: "onap-k8s-1" + user: vagrant + ssh_key_path: *ssh_key_path +services: + kube-api: + service_cluster_ip_range: 10.43.0.0/16 + pod_security_policy: false + always_pull_images: false + kube-controller: + cluster_cidr: 10.42.0.0/16 + service_cluster_ip_range: 10.43.0.0/16 + kubelet: + cluster_domain: cluster.local + cluster_dns_server: 10.43.0.10 + fail_swap_on: false +network: + plugin: canal +authentication: + strategy: x509 +ssh_key_path: *ssh_key_path +ssh_agent_auth: false +authorization: + mode: rbac +ignore_docker_version: false +kubernetes_version: "v1.13.5-rancher1-2" +private_registries: +- url: nexus3.onap.org:10001 + user: docker + password: docker + is_default: true +cluster_name: "onap" +restore: + restore: false + snapshot_name: "" diff --git a/bootstrap/vagrant-minimal-onap/tools/config/95silent-approval b/bootstrap/vagrant-minimal-onap/tools/config/95silent-approval new file mode 100644 index 000000000..dadbfbd86 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/config/95silent-approval @@ -0,0 +1,2 @@ +Quiet "1"; +APT::Get::Assume-Yes "true"; diff --git a/bootstrap/vagrant-minimal-onap/tools/config/dot_curlrc b/bootstrap/vagrant-minimal-onap/tools/config/dot_curlrc new file mode 100644 index 000000000..ecf9792f5 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/config/dot_curlrc @@ -0,0 +1,8 @@ +# Disable progress meter +--silent +# Show error messages +--show-error +# Fail silently on server errors +--fail +# Follow redirections +--location diff --git a/bootstrap/vagrant-minimal-onap/tools/config/dot_wgetrc b/bootstrap/vagrant-minimal-onap/tools/config/dot_wgetrc new file mode 100644 index 000000000..ac472b77a --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/config/dot_wgetrc @@ -0,0 +1,2 @@ +# Turn off output +quiet = on diff --git a/bootstrap/vagrant-minimal-onap/tools/get_customization_scripts.sh b/bootstrap/vagrant-minimal-onap/tools/get_customization_scripts.sh new file mode 100755 index 000000000..a99b10288 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/get_customization_scripts.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +wget \ + 'https://docs.onap.org/en/dublin/_downloads/4d5001735f875448b25f11e270d5bc5a/openstack-k8s-controlnode.sh' \ + 'https://docs.onap.org/en/dublin/_downloads/53998444dcd1b6a8b7396f7f2d35d21e/openstack-k8s-workernode.sh' diff --git a/bootstrap/vagrant-minimal-onap/tools/get_kubectl.sh b/bootstrap/vagrant-minimal-onap/tools/get_kubectl.sh new file mode 100755 index 000000000..752c286c2 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/get_kubectl.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# +# @file test/security/k8s/tools/dublin/get_kubectl.sh +# @author Pawel Wieczorek +# @brief Utility for obtaining kubectl tool +# + +# Dependencies: +# wget +# coreutils +# +# Privileges: +# Script expects to be run with administrative privileges for accessing /usr/local/bin +# +# Usage: +# # ./get_kubectl.sh [VERSION [ARCH [SYSTEM]]] +# + +# Constants +BINARY='kubectl' +INSTALL_DIR='/usr/local/bin/' + +DEFAULT_VERSION='v1.13.5' +DEFAULT_ARCH='amd64' +DEFAULT_SYSTEM='linux' + +# Variables +VERSION="${1:-$DEFAULT_VERSION}" +ARCH="${2:-$DEFAULT_ARCH}" +SYSTEM="${3:-$DEFAULT_SYSTEM}" + +URL="https://storage.googleapis.com/kubernetes-release/release/${VERSION}/bin/${SYSTEM}/${ARCH}/${BINARY}" + + +# Prerequistes +wget "$URL" +chmod +x "$BINARY" + +# Installation +mv "$BINARY" "$INSTALL_DIR" diff --git a/bootstrap/vagrant-minimal-onap/tools/get_rke.sh b/bootstrap/vagrant-minimal-onap/tools/get_rke.sh new file mode 100755 index 000000000..01dd20a96 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/get_rke.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# +# @file test/security/k8s/tools/dublin/get_rke.sh +# @author Pawel Wieczorek +# @brief Utility for obtaining RKE tool +# + +# Dependencies: +# wget +# coreutils +# +# Privileges: +# Script expects to be run with administrative privileges for accessing /usr/local/bin +# +# Usage: +# # ./get_rke.sh [VERSION [ARCH [SYSTEM]]] +# + +# Constants +DEFAULT_VERSION='v0.2.1' +DEFAULT_ARCH='amd64' +DEFAULT_SYSTEM='linux' + +# Variables +VERSION="${1:-$DEFAULT_VERSION}" +ARCH="${2:-$DEFAULT_ARCH}" +SYSTEM="${3:-$DEFAULT_SYSTEM}" + +BINARY="rke_${SYSTEM}-${ARCH}" +URL="https://github.com/rancher/rke/releases/download/${VERSION}/${BINARY}" + + +# Prerequistes +wget "$URL" +chmod +x "$BINARY" + +# Installation +mv "$BINARY" "/usr/local/bin/${BINARY%%_*}" # this also renames binary to "rke" diff --git a/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-controlnode.sh b/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-controlnode.sh new file mode 100644 index 000000000..1d230c2da --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-controlnode.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +DOCKER_VERSION=18.09.5 + +apt-get update + +curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh +mkdir -p /etc/systemd/system/docker.service.d/ +cat > /etc/systemd/system/docker.service.d/docker.conf << EOF +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// --insecure-registry=nexus3.onap.org:10001 +EOF + +sudo usermod -aG docker ubuntu + +systemctl daemon-reload +systemctl restart docker +apt-mark hold docker-ce + +IP_ADDR=`ip address |grep ens|grep inet|awk '{print $2}'| awk -F / '{print $1}'` +HOSTNAME=`hostname` + +echo "$IP_ADDR $HOSTNAME" >> /etc/hosts + +docker login -u docker -p docker nexus3.onap.org:10001 + +sudo apt-get install make -y + + +exit 0 diff --git a/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-workernode.sh b/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-workernode.sh new file mode 100644 index 000000000..3f32d050a --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/imported/openstack-k8s-workernode.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +DOCKER_VERSION=18.09.5 + +apt-get update + +curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh +mkdir -p /etc/systemd/system/docker.service.d/ +cat > /etc/systemd/system/docker.service.d/docker.conf << EOF +[Service] +ExecStart= +ExecStart=/usr/bin/dockerd -H fd:// --insecure-registry=nexus3.onap.org:10001 +EOF + +sudo usermod -aG docker ubuntu + +systemctl daemon-reload +systemctl restart docker +apt-mark hold docker-ce + +IP_ADDR=`ip address |grep ens|grep inet|awk '{print $2}'| awk -F / '{print $1}'` +HOSTNAME=`hostname` + +echo "$IP_ADDR $HOSTNAME" >> /etc/hosts + +docker login -u docker -p docker nexus3.onap.org:10001 + +sudo apt-get install make -y + +# install nfs +sudo apt-get install nfs-common -y + + +exit 0 diff --git a/bootstrap/vagrant-minimal-onap/tools/setup_kubectl.sh b/bootstrap/vagrant-minimal-onap/tools/setup_kubectl.sh new file mode 100755 index 000000000..bbd31a930 --- /dev/null +++ b/bootstrap/vagrant-minimal-onap/tools/setup_kubectl.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# +# @file test/security/k8s/tools/dublin/setup_kubectl.sh +# @author Pawel Wieczorek +# @brief Utility for setting up kubectl tool for Dublin cluster +# + +# Dependencies: +# coreutils +# +# Privileges: +# Script expects to be run with administrative privileges for accessing /usr/local/bin +# +# Usage: +# # ./setup_kubectl.sh [RKE_CONFIG [KUBE_DIR [KUBE_CONFIG [KUBE_CONTEXT]]]] +# + +# Constants +BASHRC='.bashrc' +BASH_ALIASES='.bash_aliases' +USE_ONAP_ALIAS='useonap' + +DEFAULT_RKE_CONFIG='kube_config_cluster.yml' +DEFAULT_KUBE_DIR='.kube' +DEFAULT_KUBE_CONFIG='config.onap' +DEFAULT_KUBE_CONTEXT='onap' + +# Variables +RKE_CONFIG="${1:-$DEFAULT_RKE_CONFIG}" +KUBE_DIR="${2:-${HOME}/${DEFAULT_KUBE_DIR}}" +KUBE_CONFIG="${3:-$DEFAULT_KUBE_CONFIG}" +KUBE_CONTEXT="${4:-$DEFAULT_KUBE_CONTEXT}" + +USE_ONAP="f() { export KUBECONFIG=${KUBE_DIR}/${KUBE_CONFIG}; kubectl config use-context ${KUBE_CONTEXT}; }; f" +USE_ONAP_CONFIG="$(cat<> "${HOME}/${BASH_ALIASES}" + +# Setup +cp "$RKE_CONFIG" "${KUBE_DIR}/${KUBE_CONFIG}" + +# Post-setup +echo "$USE_ONAP_CONFIG" >> "${HOME}/${BASHRC}" -- cgit 1.2.3-korg