diff options
Diffstat (limited to 'vagrant')
| -rw-r--r-- | vagrant/Vagrantfile | 14 | ||||
| -rw-r--r-- | vagrant/insecure_keys/key | 27 | ||||
| -rwxr-xr-x | vagrant/installer.sh | 109 | ||||
| -rw-r--r-- | vagrant/inventory/group_vars/k8s-cluster.yml | 13 | ||||
| -rwxr-xr-x | vagrant/main.sh | 15 | ||||
| -rw-r--r-- | vagrant/playbooks/configure-istio.yml | 2 | ||||
| -rw-r--r-- | vagrant/playbooks/configure-krd.yml | 2 | ||||
| -rw-r--r-- | vagrant/playbooks/configure-nfd.yml | 7 | ||||
| -rw-r--r-- | vagrant/playbooks/configure-virtlet.yml | 14 |
9 files changed, 122 insertions, 81 deletions
diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 8cfa4e04..1b84cb4b 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -23,7 +23,7 @@ nodes = YAML.load_file(pdf) # Inventory file creation File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file| - inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\nansible_ssh_pass=vagrant\n\n[all]") + inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\n[all]") nodes.each do |node| inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22") end @@ -59,6 +59,7 @@ end Vagrant.configure("2") do |config| config.vm.box = box[provider][:name] config.vm.box_version = box[provider][:version] + config.ssh.insert_key = false if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil if Vagrant.has_plugin?('vagrant-proxyconf') @@ -114,10 +115,15 @@ Vagrant.configure("2") do |config| config.vm.define :installer, primary: true, autostart: false do |installer| installer.vm.hostname = "multicloud" installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static - installer.vm.synced_folder '../', '/root/go/src/k8-plugin-multicloud/', type: sync_type - installer.vm.provision 'shell' do |sh| + installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type + installer.vm.provision 'shell', privileged: false do |sh| sh.env = {'KRD_PLUGIN_ENABLED': 'true'} - sh.path = "main.sh" + sh.inline = <<-SHELL + cp /vagrant/insecure_keys/key /home/vagrant/.ssh/id_rsa + chown vagrant /home/vagrant/.ssh/id_rsa + chmod 400 /home/vagrant/.ssh/id_rsa + cd /home/vagrant/multicloud-k8s/vagrant/ && ./installer.sh | tee krd_installer.log + SHELL end end end diff --git a/vagrant/insecure_keys/key b/vagrant/insecure_keys/key new file mode 100644 index 00000000..7d6a0839 --- /dev/null +++ b/vagrant/insecure_keys/key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI +w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP +kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2 +hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO +Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW +yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd +ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1 +Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf +TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK +iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A +sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf +4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP +cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk +EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN +CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX +3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG +YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj +3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+ +dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz +6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC +P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF +llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ +kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH ++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ +NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s= +-----END RSA PRIVATE KEY----- diff --git a/vagrant/installer.sh b/vagrant/installer.sh index e8ed9f11..e251170f 100755 --- a/vagrant/installer.sh +++ b/vagrant/installer.sh @@ -21,33 +21,31 @@ function _install_go { fi wget https://dl.google.com/go/$tarball - tar -C /usr/local -xzf $tarball + sudo tar -C /usr/local -xzf $tarball rm $tarball export PATH=$PATH:/usr/local/go/bin - sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment - export INSTALL_DIRECTORY=/usr/local/bin - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment } # _install_pip() - Install Python Package Manager function _install_pip { if $(pip --version &>/dev/null); then - return + sudo apt-get install -y python-dev + curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python + else + sudo -E pip install --upgrade pip fi - apt-get install -y python-dev - curl -sL https://bootstrap.pypa.io/get-pip.py | python - pip install --upgrade pip } # _install_ansible() - Install and Configure Ansible program function _install_ansible { - mkdir -p /etc/ansible/ + sudo mkdir -p /etc/ansible/ if $(ansible --version &>/dev/null); then return fi _install_pip - pip install ansible + sudo -E pip install ansible } # _install_docker() - Download and install docker-engine @@ -57,36 +55,33 @@ function _install_docker { if $(docker version &>/dev/null); then return fi - apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - apt-get update - apt-get install -y docker-ce + sudo apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get update + sudo apt-get install -y docker-ce - mkdir -p /etc/systemd/system/docker.service.d + sudo mkdir -p /etc/systemd/system/docker.service.d if [ $http_proxy ]; then - cat <<EOL > /etc/systemd/system/docker.service.d/http-proxy.conf -[Service] -Environment="HTTP_PROXY=$http_proxy" -EOL + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf + echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf fi if [ $https_proxy ]; then - cat <<EOL > /etc/systemd/system/docker.service.d/https-proxy.conf -[Service] -Environment="HTTPS_PROXY=$https_proxy" -EOL + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf + echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf fi if [ $no_proxy ]; then - cat <<EOL > /etc/systemd/system/docker.service.d/no-proxy.conf -[Service] -Environment="NO_PROXY=$no_proxy" -EOL + echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf + echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf + fi + sudo systemctl daemon-reload + echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker + if [[ -z $(groups | grep docker) ]]; then + sudo usermod -aG docker $USER + newgrp docker fi - systemctl daemon-reload - echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | tee --append /etc/default/docker - usermod -aG docker $USER - systemctl restart docker + sudo systemctl restart docker sleep 10 } @@ -95,16 +90,20 @@ function install_k8s { echo "Deploying kubernetes" local dest_folder=/opt version=$(grep "kubespray_version" ${krd_playbooks}/krd-vars.yml | awk -F ': ' '{print $2}') + local_release_dir=$(grep "local_release_dir" $krd_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}') local tarball=v$version.tar.gz - apt-get install -y sshpass + sudo apt-get install -y sshpass + _install_docker _install_ansible wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball - tar -C $dest_folder -xzf $tarball - mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo tar -C $dest_folder -xzf $tarball + sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg + sudo chown -R $USER $dest_folder/kubespray-$version + sudo mkdir -p ${local_release_dir}/containers rm $tarball - pip install -r $dest_folder/kubespray-$version/requirements.txt + sudo -E pip install -r $dest_folder/kubespray-$version/requirements.txt rm -f $krd_inventory_folder/group_vars/all.yml 2> /dev/null if [[ -n "${verbose}" ]]; then echo "kube_log_level: 5" | tee $krd_inventory_folder/group_vars/all.yml @@ -118,23 +117,23 @@ function install_k8s { if [[ -n "${https_proxy}" ]]; then echo "https_proxy: \"$https_proxy\"" | tee --append $krd_inventory_folder/group_vars/all.yml fi - ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml -b | tee $log_folder/setup-kubernetes.log + ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log # Configure environment mkdir -p $HOME/.kube - mv $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config + cp $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config } # install_addons() - Install Kubenertes AddOns function install_addons { echo "Installing Kubernetes AddOns" _install_ansible - ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors + sudo ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors - ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | tee $log_folder/setup-krd.log + ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | sudo tee $log_folder/setup-krd.log for addon in ${KRD_ADDONS:-virtlet ovn-kubernetes multus}; do echo "Deploying $addon using configure-$addon.yml playbook.." - ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | tee $log_folder/setup-${addon}.log + ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log if [[ "${testing_enabled}" == "true" ]]; then pushd $krd_tests bash ${addon}.sh @@ -148,17 +147,15 @@ function install_plugin { echo "Installing multicloud/k8s plugin" _install_go _install_docker - pip install docker-compose + sudo -E pip install docker-compose - mkdir -p /opt/{kubeconfig,consul/config} - cp $HOME/.kube/config /opt/kubeconfig/krd + sudo mkdir -p /opt/{kubeconfig,consul/config} + sudo cp $HOME/.kube/config /opt/kubeconfig/krd export KUBE_CONFIG_DIR=/opt/kubeconfig - echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" >> /etc/environment - - GOPATH=$(go env GOPATH) - pushd $GOPATH/src/k8-plugin-multicloud/deployments - ./build.sh + echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" | sudo tee --append /etc/environment + pushd $krd_folder/../deployments + sudo ./build.sh if [[ "${testing_enabled}" == "true" ]]; then docker-compose up -d pushd $krd_tests @@ -206,25 +203,25 @@ fi # Configuration values log_folder=/var/log/krd krd_folder=$(pwd) -krd_inventory_folder=$krd_folder/inventory +export krd_inventory_folder=$krd_folder/inventory krd_inventory=$krd_inventory_folder/hosts.ini krd_playbooks=$krd_folder/playbooks krd_tests=$krd_folder/tests k8s_info_file=$krd_folder/k8s_info.log testing_enabled=${KRD_ENABLE_TESTS:-false} -mkdir -p $log_folder -mkdir -p /opt/csar +sudo mkdir -p $log_folder +sudo mkdir -p /opt/csar export CSAR_DIR=/opt/csar -echo "export CSAR_DIR=${CSAR_DIR}" | tee --append /etc/environment +echo "export CSAR_DIR=${CSAR_DIR}" | sudo tee --append /etc/environment # Install dependencies # Setup proxy variables if [ -f $krd_folder/sources.list ]; then - mv /etc/apt/sources.list /etc/apt/sources.list.backup - cp $krd_folder/sources.list /etc/apt/sources.list + sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup + sudo cp $krd_folder/sources.list /etc/apt/sources.list fi -apt-get update +sudo apt-get update install_k8s install_addons if [[ "${KRD_PLUGIN_ENABLED:-false}" ]]; then diff --git a/vagrant/inventory/group_vars/k8s-cluster.yml b/vagrant/inventory/group_vars/k8s-cluster.yml index ab0c89ec..4de3a276 100644 --- a/vagrant/inventory/group_vars/k8s-cluster.yml +++ b/vagrant/inventory/group_vars/k8s-cluster.yml @@ -67,3 +67,16 @@ helm_enabled: true # works in the kernel space # https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs #kube_proxy_mode: ipvs + +# Download container images only once then push to cluster nodes in batches +download_run_once: true + +# Where the binaries will be downloaded. +# Note: ensure that you've enough disk space (about 1G) +local_release_dir: "/tmp/releases" + +# Makes the installer node a delegate for pushing images while running +# the deployment with ansible. This maybe the case if cluster nodes +# cannot access each over via ssh or you want to use local docker +# images as a cache for multiple clusters. +download_localhost: true diff --git a/vagrant/main.sh b/vagrant/main.sh deleted file mode 100755 index 993ca78a..00000000 --- a/vagrant/main.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -############################################################################## -# Copyright (c) 2018 -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -o nounset -set -o pipefail - -cd ~/go/src/k8-plugin-multicloud/vagrant -sudo -H -E bash ./installer.sh | tee krd_installer.log diff --git a/vagrant/playbooks/configure-istio.yml b/vagrant/playbooks/configure-istio.yml index e6a138e7..2bd4e853 100644 --- a/vagrant/playbooks/configure-istio.yml +++ b/vagrant/playbooks/configure-istio.yml @@ -9,7 +9,6 @@ ############################################################################## - hosts: localhost - become: yes pre_tasks: - name: Load krd variables include_vars: @@ -36,6 +35,7 @@ dest: "{{ istio_dest }}" remote_src: yes - name: copy istioctl binary to usr/local/bin folder + become: yes command: "mv {{ istio_dest }}/istio-{{ istio_version }}/bin/istioctl /usr/local/bin/" when: istio_source_type == "tarball" - name: create network objects diff --git a/vagrant/playbooks/configure-krd.yml b/vagrant/playbooks/configure-krd.yml index c8146ed8..22e6419f 100644 --- a/vagrant/playbooks/configure-krd.yml +++ b/vagrant/playbooks/configure-krd.yml @@ -12,5 +12,5 @@ tasks: - name: copy admin.conf file to kube-nodes copy: - src: "{{ ansible_env.HOME}}/.kube/config" + src: "{{ lookup('env','krd_inventory_folder') }}/artifacts/admin.conf" dest: "/etc/kubernetes/admin.conf" diff --git a/vagrant/playbooks/configure-nfd.yml b/vagrant/playbooks/configure-nfd.yml index 26ad5497..d47a7bcc 100644 --- a/vagrant/playbooks/configure-nfd.yml +++ b/vagrant/playbooks/configure-nfd.yml @@ -46,10 +46,13 @@ - node-feature-discovery-daemonset.json.template - hosts: localhost - become: yes + pre_tasks: + - name: Load krd variables + include_vars: + file: krd-vars.yml roles: - role: andrewrothstein.kubectl - kubectl_ver: "v{{ kubectl_version }} + kubectl_ver: "v{{ kubectl_version }}" tasks: - name: create service accounts command: "/usr/local/bin/kubectl apply -f /tmp/{{ item }}" diff --git a/vagrant/playbooks/configure-virtlet.yml b/vagrant/playbooks/configure-virtlet.yml index b1dee09f..66deb5cb 100644 --- a/vagrant/playbooks/configure-virtlet.yml +++ b/vagrant/playbooks/configure-virtlet.yml @@ -8,7 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - hosts: localhost - become: yes vars: images_file: /tmp/images.yaml pre_tasks: @@ -67,10 +66,12 @@ - name: configure proxy values for docker service block: - name: create docker config folder + become: yes file: state: directory path: "/etc/systemd/system/docker.service.d" - name: Configure docker service to use http_proxy env value + become: yes blockinfile: dest: "/etc/systemd/system/docker.service.d/http-proxy.conf" create: yes @@ -80,6 +81,7 @@ when: - lookup('env','http_proxy') != "fooproxy" - name: Configure docker service to use https_proxy env value + become: yes blockinfile: dest: "/etc/systemd/system/docker.service.d/https-proxy.conf" create: yes @@ -89,6 +91,7 @@ when: - lookup('env','https_proxy') != "fooproxy" - name: Configure docker service to use no_proxy env value + become: yes blockinfile: dest: "/etc/systemd/system/docker.service.d/no-proxy.conf" create: yes @@ -98,8 +101,10 @@ when: - lookup('env','no_proxy') != "fooproxy" - name: reload systemd + become: yes command: systemctl daemon-reload - name: restart docker service + become: yes service: name: docker state: restarted @@ -136,7 +141,6 @@ delay: 10 - hosts: virtlet - become: yes tasks: - name: Load krd variables include_vars: @@ -146,18 +150,21 @@ state: directory path: "{{ criproxy_dest }}" - name: disable AppArmor in all nodes + become: yes service: name: apparmor state: stopped enabled: no when: ansible_os_family == "Debian" - name: modify args for kubelet service + become: yes lineinfile: dest: /etc/systemd/system/kubelet.service line: " --container-runtime=remote --container-runtime-endpoint=unix:///run/criproxy.sock --image-service-endpoint=unix:///run/criproxy.sock --enable-controller-attach-detach=false \\" insertafter: '^ExecStart=/usr/local/bin/kubelet *' state: present - name: create dockershim service + become: yes blockinfile: path: /etc/systemd/system/dockershim.service create: yes @@ -210,6 +217,7 @@ path: "{{ criproxy_dest }}/criproxy" mode: "+x" - name: create criproxy service + become: yes blockinfile: path: /etc/systemd/system/criproxy.service create: yes @@ -226,6 +234,7 @@ [Install] WantedBy=kubelet.service - name: start criproxy and dockershim services + become: yes service: name: "{{ item }}" state: started @@ -234,6 +243,7 @@ - dockershim - criproxy - name: restart kubelet services + become: yes service: name: kubelet state: restarted |