Pull images from csv file

It has been included another method for retrieving OOM images. This
function uses a cvs file to get the information to previously loads
the Docker registry for Helm Charts deployment.

Change-Id: I2e6d154cd7911d65b44592f9cf512180bb05bef6
Signed-off-by: Victor Morales <victor.morales@intel.com>
Issue-ID: INT-478

13 files changed, 72 insertions, 43 deletions

diff --git a/Vagrantfile b/Vagrantfile
index f10e8b0..ca6bb8d 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -161,17 +161,12 @@ nodes = [
     :name   => "oom",
     :ips    => ['10.252.0.21', "192.168.50.21"],
     :macs   => [],
-    :cpus   => 16,
+    :cpus   => 32,
     :cpu    => "50",
-    :ram    => 70 * 1024,
+    :ram    => 128 * 1024,
     :groups => ["individual"],
     :args   => ["oom"],
     :hd     => { :virtualbox => "163840", :libvirt => "160G", },
-    :fwds   => [
-      { :guest => 8880, :host => 8880, :guest_ip => '192.168.50.21' },
-      { :guest => 8989, :host => 8989, :guest_ip => '192.168.50.21' },
-      { :guest => 8080, :host => 8888, :guest_ip => '192.168.50.21' },
-    ]
   },
   {
     :name   => "policy",
diff --git a/lib/_installers b/lib/_installers
index bfc8574..f972d36 100755
--- a/lib/_installers
+++ b/lib/_installers
@@ -350,10 +350,9 @@ function _install_kubespray_k8s {
     local src_folder=/opt/kubespray
 
     clone_repo kubernetes-incubator/kubespray $src_folder https://github.com/
-    install_python_package netaddr
-    install_python_package ansible==2.4.0.0
     install_docker
     pushd $src_folder
+    install_python_requirements .
     rm -rf inventory/*
     mkdir -p inventory/group_vars
     cp /var/onap/files/aio_inventory.cfg ./inventory/inventory.cfg
diff --git a/lib/_onap_functions b/lib/_onap_functions
index 29f6bbd..c5ee7fc 100755
--- a/lib/_onap_functions
+++ b/lib/_onap_functions
@@ -25,9 +25,10 @@ function docker_openecomp_login {
 # pull_openecomp_image() - Pull Docker container image from a Docker Registry Hub
 function pull_openecomp_image {
     local image=$1
-    local tag=$2
+    local version=${2:-$docker_version}
+    local tag=$3
     docker_openecomp_login
-    pull_docker_image ${nexus_docker_repo:-nexus3.onap.org:10001}/openecomp/${image}:${docker_version:-latest} $tag
+    pull_docker_image ${nexus_docker_repo:-nexus3.onap.org:10001}/openecomp/${image}:${version} $tag
     docker logout
 }
 
diff --git a/lib/appc b/lib/appc
index 95654fc..4f6e037 100755
--- a/lib/appc
+++ b/lib/appc
@@ -15,7 +15,7 @@ function get_appc_images {
         _build_appc_images
     else
         for image in appc-image dgbuilder-sdnc-image; do
-            pull_openecomp_image $image openecomp/$image:latest
+            pull_openecomp_image $image ${docker_version:-latest} openecomp/$image:latest
         done
     fi
 }
diff --git a/lib/config/env-vars b/lib/config/env-vars
index c448321..a3a4c33 100755
--- a/lib/config/env-vars
+++ b/lib/config/env-vars
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# URL repository
+git_url="https://git.onap.org/"
+
 # Source code destination folder
 git_src_folder=/opt/onap
 
diff --git a/lib/dcae b/lib/dcae
index 25efddd..93b2ac8 100755
--- a/lib/dcae
+++ b/lib/dcae
@@ -51,8 +51,8 @@ function get_dcae_images {
     if [[ "$build_image" == "True" ]]; then
         _build_dcae_images
     else
-        pull_openecomp_image dcae-dmaapbc openecomp/dcae-dmaapbc
-        pull_openecomp_image dcae-controller
+        pull_openecomp_image dcae-dmaapbc ${docker_version:-latest} openecomp/dcae-dmaapbc
+        pull_openecomp_image dcae-controller ${docker_version:-latest}
     fi
 }
 
diff --git a/lib/files/k8s-cluster.yml b/lib/files/k8s-cluster.yml
index 0e6f6c0..3579e7a 100644
--- a/lib/files/k8s-cluster.yml
+++ b/lib/files/k8s-cluster.yml
@@ -32,7 +32,7 @@ bin_dir: /usr/local/bin
 ## for mounting persistent volumes into containers.  These may not be loaded by preinstall kubernetes
 ## processes.  For example, ceph and rbd backed volumes.  Set to true to allow kubelet to load kernel
 ## modules.
-# kubelet_load_modules: false
+kubelet_load_modules: true
 
 ## Internal network total size. This is the prefix of the
 ## entire network. Must be unused in your environment.
@@ -147,7 +147,7 @@ kube_log_level: 2
 
 # Users to create for basic auth in Kubernetes API via HTTP
 # Optionally add groups for user
-kube_api_pwd: "{{ lookup('password', 'credentials/kube_user length=15 chars=ascii_letters,digits') }}"
+kube_api_pwd: "secret"
 kube_users:
   kube:
     pass: "{{kube_api_pwd}}"
@@ -224,7 +224,7 @@ dns_mode: kubedns
 # Can be docker_dns, host_resolvconf or none
 resolvconf_mode: docker_dns
 # Deploy netchecker app to verify DNS resolve as an HTTP service
-deploy_netchecker: false
+deploy_netchecker: true
 # Ip address of the kubernetes skydns service
 skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
@@ -255,7 +255,7 @@ k8s_image_pull_policy: IfNotPresent
 dashboard_enabled: true
 
 # Monitoring apps for k8s
-efk_enabled: false
+efk_enabled: true
 
 # Helm deployment
 helm_enabled: false
diff --git a/lib/functions b/lib/functions
index 265a34c..198074b 100755
--- a/lib/functions
+++ b/lib/functions
@@ -52,7 +52,7 @@ function _git_timed {
 
 # clone_repo() - Clone Git repository into specific folder
 function clone_repo {
-    local repo_url=${3:-"https://git.onap.org/"}
+    local repo_url=${3:-$git_url}
     local repo=$1
     local dest_folder=${2:-$git_src_folder/$repo}
     if [ ! -d $dest_folder ]; then
@@ -79,6 +79,8 @@ function pull_docker_image {
     install_docker
     local image=$1
     local tag=$2
+
+    echo "docker pull ${image}"
     docker pull ${image}
     if [ ${tag} ]; then
         docker tag ${image} $tag
diff --git a/lib/oom b/lib/oom
index 91df0a9..ea0bae0 100755
--- a/lib/oom
+++ b/lib/oom
@@ -3,9 +3,24 @@
 source /var/onap/functions
 
 oom_delay=30
+k8s_deployment_tool="kubespray"
 
 # _pull_images_from_yaml() - Function that parses a yaml file and pull their images
 function _pull_images_from_yaml_file {
+    if [[ "$clone_repo" != "True" ]]; then
+        clone_repos "oom"
+    fi
+
+    docker_openecomp_login
+    for values_file in `find ${src_folders[oom]}/kubernetes -name values.yaml -type f`; do
+        _parse_yaml_file $values_file
+    done
+    docker logout
+    wait_docker_pull
+}
+
+# _parse_yaml_file() - Fuction that parses the OOM Charts files
+function _parse_yaml_file {
     local values_file=$1
     local prefix=$2
     local s='[[:space:]]*'
@@ -41,33 +56,38 @@ for (i in vname) {
     done
 }
 
+# _pull_images_from_csv_file() - Function that parses a csv file to pull their images
+function _pull_images_from_csv_file {
+    local filename=docker-manifest.csv
+
+    wget ${git_url}/integration/plain/version-manifest/src/main/resources/$filename
+
+    install_docker
+    for line in $(tail -n +2 $filename); do
+        image=${line%,*}
+        pull_$(echo $image | cut -d / -f 1)_image ${image#*/} ${line#*,} &
+    done
+    rm $filename
+    wait_docker_pull
+}
+
 # get_oom_images() - Function that retrieves ONAP images from official hub
 function get_oom_images {
+    local single_source_of_truth=${1:-"csv"}
+
     if [[ "$build_image" == "True" ]]; then
         for project in aai appc dcae dmaap mr msb mso multicloud policy portal robot sdc sdnc vfc vid; do
             source /var/onap/$project
             get_${project}_images
         done
     else
-        if [[ "$clone_repo" != "True" ]]; then
-            clone_repos "oom"
-        fi
-
-        docker_openecomp_login
-        for values_file in `find ${src_folders[oom]}/kubernetes -name values.yaml -type f`; do
-            _pull_images_from_yaml_file $values_file
-        done
-        docker logout
-        wait_docker_pull
+        _pull_images_from_${single_source_of_truth}_file
     fi
 }
 
 # install_oom() - Function that clones OOM and deploys ONAP
 function install_oom {
-    local k8s_deployment_tool="kubespray"
-
-    mount_external_partition sda /var/lib/docker/
-    install_kubernetes kubespray
+    install_kubernetes $k8s_deployment_tool
     if [[ "$deployment_tool" == "rancher" ]]; then
         pushd ~/.kube
         install_python_package requests
@@ -79,10 +99,6 @@ function install_oom {
         sleep $oom_delay
     done
     install_helm
-    printf "Kubernetes Info\n===============\n" > k8s_info.log
-    echo "Dashboard URL: http://$IP_ADDRESS:$(kubectl get service -n kube-system |grep kubernetes-dashboard | awk '{print $5}' |awk -F "[:/]" '{print $2}')" >> k8s_info.log
-    echo "Admin user: $(cat /etc/kubernetes/users/known_users.csv |awk -F ',' '{print $2}')" >> k8s_info.log
-    echo "Admin password: $(cat /etc/kubernetes/users/known_users.csv |awk -F ',' '{print $1}')" >> k8s_info.log
 
     if [[ "$clone_repo" != "True" ]]; then
         clone_repos "oom"
@@ -91,10 +107,14 @@ function install_oom {
     make repo
     make all
     helm install local/onap -n beijing -f /var/onap/files/dev.yaml
+    popd
 }
 
 # init_oom() - Function that deploys ONAP using OOM
 function init_oom {
+    local k8s_info_file=/var/log/k8s_info.log
+    mount_external_partition sda /var/lib/docker/
+
     if [[ "$clone_repo" == "True" ]]; then
         clone_repos "oom"
     fi
@@ -103,5 +123,12 @@ function init_oom {
     fi
     if [[ "$skip_install" == "False" ]]; then
         install_oom
+        if [[ "$k8s_deployment_tool" == "kubespray" ]]; then
+            printf "Kubernetes Info\n===============\n" > $k8s_info_file
+            echo "Dashboard URL: https://$IP_ADDRESS:$(kubectl get service -n kube-system |grep kubernetes-dashboard | awk '{print $5}' |awk -F "[:/]" '{print $2}')" >> $k8s_info_file
+            echo "Admin user: $(cat /etc/kubernetes/users/known_users.csv |awk -F ',' '{print $2}')" >> $k8s_info_file
+            echo "Admin password: $(cat /etc/kubernetes/users/known_users.csv |awk -F ',' '{print $1}')" >> $k8s_info_file
+            cat $k8s_info_file
+        fi
     fi
 }
diff --git a/lib/portal b/lib/portal
index fe54698..0d59dab 100755
--- a/lib/portal
+++ b/lib/portal
@@ -44,8 +44,8 @@ function get_portal_images {
     if [[ "$build_image" == "True" ]]; then
         _build_portal_images
     else
-        pull_openecomp_image portaldb ecompdb:portal
-        pull_openecomp_image portalapps ep:1610-1
+        pull_openecomp_image portaldb ${docker_version:-latest} ecompdb:portal
+        pull_openecomp_image portalapps ${docker_version:-latest} ep:1610-1
     fi
     pull_docker_image mariadb
 }
diff --git a/lib/sdnc b/lib/sdnc
index 8dacf9e..3d1d592 100755
--- a/lib/sdnc
+++ b/lib/sdnc
@@ -34,7 +34,7 @@ function get_sdnc_images {
         _build_sdnc_images
     else
         for image in sdnc-image admportal-sdnc-image dgbuilder-sdnc-image; do
-            pull_openecomp_image $image openecomp/$image:latest
+            pull_openecomp_image $image ${docker_version:-latest} openecomp/$image:latest
         done
     fi
     pull_docker_image mysql/mysql-server:5.6
diff --git a/lib/vid b/lib/vid
index 0c7ad85..331dc2b 100755
--- a/lib/vid
+++ b/lib/vid
@@ -15,7 +15,7 @@ function get_vid_images {
     if [[ "$build_image" == "True" ]]; then
         _build_vid_images
     else
-        pull_openecomp_image vid
+        pull_openecomp_image vid ${docker_version:-latest}
     fi
     pull_docker_image mariadb:10
 }
diff --git a/tools/setup.sh b/tools/setup.sh
index 8b0ef42..54d10bf 100755
--- a/tools/setup.sh
+++ b/tools/setup.sh
@@ -8,11 +8,13 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-vagrant_version=2.0.3
+vagrant_version=2.1.1
 
 function usage {
     cat <<EOF
-usage: setup.sh -p <PROVIDER>
+usage: $0 -p <PROVIDER>
+Installation of vagrant and its dependencies in Linux OS
+
 Argument:
     -p  Vagrant provider
 EOF