Merge "Enable downloading binaries and containers"

9 files changed, 122 insertions, 81 deletions

diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile
index 8cfa4e04..1b84cb4b 100644
--- a/vagrant/Vagrantfile
+++ b/vagrant/Vagrantfile
@@ -23,7 +23,7 @@ nodes = YAML.load_file(pdf)
 
 # Inventory file creation
 File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file|
-  inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\nansible_ssh_pass=vagrant\n\n[all]")
+  inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\n[all]")
   nodes.each do |node|
     inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22")
   end
@@ -59,6 +59,7 @@ end
 Vagrant.configure("2") do |config|
   config.vm.box =  box[provider][:name]
   config.vm.box_version = box[provider][:version]
+  config.ssh.insert_key = false
 
   if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil
     if Vagrant.has_plugin?('vagrant-proxyconf')
@@ -114,10 +115,15 @@ Vagrant.configure("2") do |config|
   config.vm.define :installer, primary: true, autostart: false do |installer|
     installer.vm.hostname = "multicloud"
     installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static
-    installer.vm.synced_folder '../', '/root/go/src/k8-plugin-multicloud/', type: sync_type
-    installer.vm.provision 'shell' do |sh|
+    installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type
+    installer.vm.provision 'shell', privileged: false do |sh|
       sh.env = {'KRD_PLUGIN_ENABLED': 'true'}
-      sh.path = "main.sh"
+      sh.inline = <<-SHELL
+        cp /vagrant/insecure_keys/key /home/vagrant/.ssh/id_rsa
+        chown vagrant /home/vagrant/.ssh/id_rsa
+        chmod 400 /home/vagrant/.ssh/id_rsa
+        cd /home/vagrant/multicloud-k8s/vagrant/ && ./installer.sh | tee krd_installer.log
+      SHELL
     end
   end
 end
diff --git a/vagrant/insecure_keys/key b/vagrant/insecure_keys/key
new file mode 100644
index 00000000..7d6a0839
--- /dev/null
+++ b/vagrant/insecure_keys/key
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----
diff --git a/vagrant/installer.sh b/vagrant/installer.sh
index e8ed9f11..e251170f 100755
--- a/vagrant/installer.sh
+++ b/vagrant/installer.sh
@@ -21,33 +21,31 @@ function _install_go {
     fi
 
     wget https://dl.google.com/go/$tarball
-    tar -C /usr/local -xzf $tarball
+    sudo tar -C /usr/local -xzf $tarball
     rm $tarball
 
     export PATH=$PATH:/usr/local/go/bin
-    sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment
-    export INSTALL_DIRECTORY=/usr/local/bin
-    curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+    sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment
 }
 
 # _install_pip() - Install Python Package Manager
 function _install_pip {
     if $(pip --version &>/dev/null); then
-        return
+        sudo apt-get install -y python-dev
+        curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python
+    else
+        sudo -E pip install --upgrade pip
     fi
-    apt-get install -y python-dev
-    curl -sL https://bootstrap.pypa.io/get-pip.py | python
-    pip install --upgrade pip
 }
 
 # _install_ansible() - Install and Configure Ansible program
 function _install_ansible {
-    mkdir -p /etc/ansible/
+    sudo mkdir -p /etc/ansible/
     if $(ansible --version &>/dev/null); then
         return
     fi
     _install_pip
-    pip install ansible
+    sudo -E pip install ansible
 }
 
 # _install_docker() - Download and install docker-engine
@@ -57,36 +55,33 @@ function _install_docker {
     if $(docker version &>/dev/null); then
         return
     fi
-    apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl
-    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-    apt-get update
-    apt-get install -y docker-ce
+    sudo apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    sudo apt-get update
+    sudo apt-get install -y docker-ce
 
-    mkdir -p /etc/systemd/system/docker.service.d
+    sudo mkdir -p /etc/systemd/system/docker.service.d
     if [ $http_proxy ]; then
-        cat <<EOL > /etc/systemd/system/docker.service.d/http-proxy.conf
-[Service]
-Environment="HTTP_PROXY=$http_proxy"
-EOL
+        echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf
+        echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf
     fi
     if [ $https_proxy ]; then
-        cat <<EOL > /etc/systemd/system/docker.service.d/https-proxy.conf
-[Service]
-Environment="HTTPS_PROXY=$https_proxy"
-EOL
+        echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf
+        echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf
     fi
     if [ $no_proxy ]; then
-        cat <<EOL > /etc/systemd/system/docker.service.d/no-proxy.conf
-[Service]
-Environment="NO_PROXY=$no_proxy"
-EOL
+        echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf
+        echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf
+    fi
+    sudo systemctl daemon-reload
+    echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker
+    if [[ -z $(groups | grep docker) ]]; then
+        sudo usermod -aG docker $USER
+        newgrp docker
     fi
-    systemctl daemon-reload
-    echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | tee --append /etc/default/docker
-    usermod -aG docker $USER
 
-    systemctl restart docker
+    sudo systemctl restart docker
     sleep 10
 }
 
@@ -95,16 +90,20 @@ function install_k8s {
     echo "Deploying kubernetes"
     local dest_folder=/opt
     version=$(grep "kubespray_version" ${krd_playbooks}/krd-vars.yml | awk -F ': ' '{print $2}')
+    local_release_dir=$(grep "local_release_dir" $krd_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}')
     local tarball=v$version.tar.gz
 
-    apt-get install -y sshpass
+    sudo apt-get install -y sshpass
+    _install_docker
     _install_ansible
     wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball
-    tar -C $dest_folder -xzf $tarball
-    mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
+    sudo tar -C $dest_folder -xzf $tarball
+    sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
+    sudo chown -R $USER $dest_folder/kubespray-$version
+    sudo mkdir -p ${local_release_dir}/containers
     rm $tarball
 
-    pip install -r $dest_folder/kubespray-$version/requirements.txt
+    sudo -E pip install -r $dest_folder/kubespray-$version/requirements.txt
     rm -f $krd_inventory_folder/group_vars/all.yml 2> /dev/null
     if [[ -n "${verbose}" ]]; then
         echo "kube_log_level: 5" | tee $krd_inventory_folder/group_vars/all.yml
@@ -118,23 +117,23 @@ function install_k8s {
     if [[ -n "${https_proxy}" ]]; then
         echo "https_proxy: \"$https_proxy\"" | tee --append $krd_inventory_folder/group_vars/all.yml
     fi
-    ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml -b | tee $log_folder/setup-kubernetes.log
+    ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log
 
     # Configure environment
     mkdir -p $HOME/.kube
-    mv $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config
+    cp $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config
 }
 
 # install_addons() - Install Kubenertes AddOns
 function install_addons {
     echo "Installing Kubernetes AddOns"
     _install_ansible
-    ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors
+    sudo ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors
 
-    ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | tee $log_folder/setup-krd.log
+    ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | sudo tee $log_folder/setup-krd.log
     for addon in ${KRD_ADDONS:-virtlet ovn-kubernetes multus}; do
         echo "Deploying $addon using configure-$addon.yml playbook.."
-        ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | tee $log_folder/setup-${addon}.log
+        ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log
         if [[ "${testing_enabled}" == "true" ]]; then
             pushd $krd_tests
             bash ${addon}.sh
@@ -148,17 +147,15 @@ function install_plugin {
     echo "Installing multicloud/k8s plugin"
     _install_go
     _install_docker
-    pip install docker-compose
+    sudo -E pip install docker-compose
 
-    mkdir -p /opt/{kubeconfig,consul/config}
-    cp $HOME/.kube/config /opt/kubeconfig/krd
+    sudo mkdir -p /opt/{kubeconfig,consul/config}
+    sudo cp $HOME/.kube/config /opt/kubeconfig/krd
     export KUBE_CONFIG_DIR=/opt/kubeconfig
-    echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" >> /etc/environment
-
-    GOPATH=$(go env GOPATH)
-    pushd $GOPATH/src/k8-plugin-multicloud/deployments
-    ./build.sh
+    echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" | sudo tee --append /etc/environment
 
+    pushd $krd_folder/../deployments
+    sudo ./build.sh
     if [[ "${testing_enabled}" == "true" ]]; then
         docker-compose up -d
         pushd $krd_tests
@@ -206,25 +203,25 @@ fi
 # Configuration values
 log_folder=/var/log/krd
 krd_folder=$(pwd)
-krd_inventory_folder=$krd_folder/inventory
+export krd_inventory_folder=$krd_folder/inventory
 krd_inventory=$krd_inventory_folder/hosts.ini
 krd_playbooks=$krd_folder/playbooks
 krd_tests=$krd_folder/tests
 k8s_info_file=$krd_folder/k8s_info.log
 testing_enabled=${KRD_ENABLE_TESTS:-false}
 
-mkdir -p $log_folder
-mkdir -p /opt/csar
+sudo mkdir -p $log_folder
+sudo mkdir -p /opt/csar
 export CSAR_DIR=/opt/csar
-echo "export CSAR_DIR=${CSAR_DIR}" | tee --append /etc/environment
+echo "export CSAR_DIR=${CSAR_DIR}" | sudo tee --append /etc/environment
 
 # Install dependencies
 # Setup proxy variables
 if [ -f $krd_folder/sources.list ]; then
-    mv /etc/apt/sources.list /etc/apt/sources.list.backup
-    cp $krd_folder/sources.list /etc/apt/sources.list
+    sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup
+    sudo cp $krd_folder/sources.list /etc/apt/sources.list
 fi
-apt-get update
+sudo apt-get update
 install_k8s
 install_addons
 if [[ "${KRD_PLUGIN_ENABLED:-false}" ]]; then
diff --git a/vagrant/inventory/group_vars/k8s-cluster.yml b/vagrant/inventory/group_vars/k8s-cluster.yml
index ab0c89ec..4de3a276 100644
--- a/vagrant/inventory/group_vars/k8s-cluster.yml
+++ b/vagrant/inventory/group_vars/k8s-cluster.yml
@@ -67,3 +67,16 @@ helm_enabled: true
 # works in the kernel space
 # https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs
 #kube_proxy_mode: ipvs
+
+# Download container images only once then push to cluster nodes in batches
+download_run_once: true
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+
+# Makes the installer node a delegate for pushing images while running
+# the deployment with ansible. This maybe the case if cluster nodes
+# cannot access each over via ssh or you want to use local docker
+# images as a cache for multiple clusters.
+download_localhost: true
diff --git a/vagrant/main.sh b/vagrant/main.sh
deleted file mode 100755
index 993ca78a..00000000
--- a/vagrant/main.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-##############################################################################
-# Copyright (c) 2018
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-set -o nounset
-set -o pipefail
-
-cd ~/go/src/k8-plugin-multicloud/vagrant
-sudo -H -E bash ./installer.sh | tee krd_installer.log
diff --git a/vagrant/playbooks/configure-istio.yml b/vagrant/playbooks/configure-istio.yml
index e6a138e7..2bd4e853 100644
--- a/vagrant/playbooks/configure-istio.yml
+++ b/vagrant/playbooks/configure-istio.yml
@@ -9,7 +9,6 @@
 ##############################################################################
 
 - hosts: localhost
-  become: yes
   pre_tasks:
     - name: Load krd variables
       include_vars:
@@ -36,6 +35,7 @@
           dest: "{{ istio_dest }}"
           remote_src: yes
       - name: copy istioctl binary to usr/local/bin folder
+        become: yes
         command: "mv {{ istio_dest }}/istio-{{ istio_version }}/bin/istioctl /usr/local/bin/"
       when: istio_source_type == "tarball"
     - name: create network objects
diff --git a/vagrant/playbooks/configure-krd.yml b/vagrant/playbooks/configure-krd.yml
index c8146ed8..22e6419f 100644
--- a/vagrant/playbooks/configure-krd.yml
+++ b/vagrant/playbooks/configure-krd.yml
@@ -12,5 +12,5 @@
   tasks:
     - name: copy admin.conf file to kube-nodes
       copy:
-        src: "{{ ansible_env.HOME}}/.kube/config"
+        src: "{{ lookup('env','krd_inventory_folder') }}/artifacts/admin.conf"
         dest: "/etc/kubernetes/admin.conf"
diff --git a/vagrant/playbooks/configure-nfd.yml b/vagrant/playbooks/configure-nfd.yml
index 26ad5497..d47a7bcc 100644
--- a/vagrant/playbooks/configure-nfd.yml
+++ b/vagrant/playbooks/configure-nfd.yml
@@ -46,10 +46,13 @@
         - node-feature-discovery-daemonset.json.template
 
 - hosts: localhost
-  become: yes
+  pre_tasks:
+    - name: Load krd variables
+      include_vars:
+        file: krd-vars.yml
   roles:
     - role: andrewrothstein.kubectl
-      kubectl_ver: "v{{ kubectl_version }}
+      kubectl_ver: "v{{ kubectl_version }}"
   tasks:
     - name: create service accounts
       command: "/usr/local/bin/kubectl apply -f /tmp/{{ item }}"
diff --git a/vagrant/playbooks/configure-virtlet.yml b/vagrant/playbooks/configure-virtlet.yml
index b1dee09f..66deb5cb 100644
--- a/vagrant/playbooks/configure-virtlet.yml
+++ b/vagrant/playbooks/configure-virtlet.yml
@@ -8,7 +8,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 - hosts: localhost
-  become: yes
   vars:
     images_file: /tmp/images.yaml
   pre_tasks:
@@ -67,10 +66,12 @@
       - name: configure proxy values for docker service
         block:
         - name: create docker config folder
+          become: yes
           file:
             state: directory
             path: "/etc/systemd/system/docker.service.d"
         - name: Configure docker service to use http_proxy env value
+          become: yes
           blockinfile:
             dest: "/etc/systemd/system/docker.service.d/http-proxy.conf"
             create: yes
@@ -80,6 +81,7 @@
           when:
             - lookup('env','http_proxy') != "fooproxy"
         - name: Configure docker service to use https_proxy env value
+          become: yes
           blockinfile:
             dest: "/etc/systemd/system/docker.service.d/https-proxy.conf"
             create: yes
@@ -89,6 +91,7 @@
           when:
             - lookup('env','https_proxy') != "fooproxy"
         - name: Configure docker service to use no_proxy env value
+          become: yes
           blockinfile:
             dest: "/etc/systemd/system/docker.service.d/no-proxy.conf"
             create: yes
@@ -98,8 +101,10 @@
           when:
             - lookup('env','no_proxy') != "fooproxy"
         - name: reload systemd
+          become: yes
           command: systemctl daemon-reload
         - name: restart docker service
+          become: yes
           service:
             name: docker
             state: restarted
@@ -136,7 +141,6 @@
       delay: 10
 
 - hosts: virtlet
-  become: yes
   tasks:
     - name: Load krd variables
       include_vars:
@@ -146,18 +150,21 @@
         state: directory
         path: "{{ criproxy_dest }}"
     - name: disable AppArmor in all nodes
+      become: yes
       service:
         name: apparmor
         state: stopped
         enabled: no
       when: ansible_os_family == "Debian"
     - name: modify args for kubelet service
+      become: yes
       lineinfile:
         dest: /etc/systemd/system/kubelet.service
         line: "                --container-runtime=remote --container-runtime-endpoint=unix:///run/criproxy.sock --image-service-endpoint=unix:///run/criproxy.sock --enable-controller-attach-detach=false \\"
         insertafter: '^ExecStart=/usr/local/bin/kubelet *'
         state: present
     - name: create dockershim service
+      become: yes
       blockinfile:
         path: /etc/systemd/system/dockershim.service
         create: yes
@@ -210,6 +217,7 @@
         path: "{{ criproxy_dest }}/criproxy"
         mode: "+x"
     - name: create criproxy service
+      become: yes
       blockinfile:
         path: /etc/systemd/system/criproxy.service
         create: yes
@@ -226,6 +234,7 @@
           [Install]
           WantedBy=kubelet.service
     - name: start criproxy and dockershim services
+      become: yes
       service:
         name: "{{ item }}"
         state: started
@@ -234,6 +243,7 @@
       - dockershim
       - criproxy
     - name: restart kubelet services
+      become: yes
       service:
         name: kubelet
         state: restarted