Create a upstream repos validator

This Unit Test helps to validate the usage of the official
repositrories and fails in case that a minimum are not used.

Change-Id: Id5a922163a8a0d16de5460aee92e5cced3f285a3
Signed-off-by: Victor Morales <victor.morales@intel.com>
Issue-Id: INT-289

14 files changed, 311 insertions, 58 deletions

diff --git a/.gitignore b/.gitignore
index f3b07b8d9..48c5c1c21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ env.properties
 .vagrant
 *~
 .checkstyle
+.DS_Store
diff --git a/bootstrap/vagrant-onap/.gitignore b/bootstrap/vagrant-onap/.gitignore
index 3c502a0af..873bd8b4d 100644
--- a/bootstrap/vagrant-onap/.gitignore
+++ b/bootstrap/vagrant-onap/.gitignore
@@ -8,3 +8,5 @@ lib/files/sources.list
 openrc
 doc/build/
 etc/settings.yaml
+tests/remaining_projects.txt
+tests/installed-software*
diff --git a/bootstrap/vagrant-onap/lib/appc b/bootstrap/vagrant-onap/lib/appc
index 5e3858f16..4d93f54e5 100755
--- a/bootstrap/vagrant-onap/lib/appc
+++ b/bootstrap/vagrant-onap/lib/appc
@@ -6,17 +6,20 @@ source /var/onap/sdnc
 source /var/onap/functions
 
 appc_src_folder=$git_src_folder/appc
+appc_repos=("appc" "appc/deployment")
 
 # clone_all_appc_repos() - Function that clones APPC source repo.
 function clone_all_appc_repos {
-    clone_repo appc $appc_src_folder
-    clone_repo appc/deployment $appc_src_folder/deployment
+    for repo in ${appc_repos[@]}; do
+        clone_repo $repo $appc_src_folder${repo#*appc}
+    done
 }
 
 # compile_all_appc_repos() - Function that compiles APPC source repo.
 function compile_all_appc_repos {
-    compile_src $appc_src_folder
-    compile_src $appc_src_folder/deployment
+    for repo in ${appc_repos[@]}; do
+        compile_src $appc_src_folder${repo#*appc}
+    done
 }
 
 # _build_appc_images() - Function that creates APPC images from source code.
diff --git a/bootstrap/vagrant-onap/lib/ccsdk b/bootstrap/vagrant-onap/lib/ccsdk
index 85cdf4da2..03987ea7e 100755
--- a/bootstrap/vagrant-onap/lib/ccsdk
+++ b/bootstrap/vagrant-onap/lib/ccsdk
@@ -5,23 +5,24 @@ set -o xtrace
 source /var/onap/functions
 
 ccsdk_src_folder=$git_src_folder/ccsdk
-ccsdk_repos=("dashboard" "distribution" "parent" "platform/blueprints" "platform/nbapi" "platform/plugins" "sli" "sli/adaptors" "sli/core" "sli/northbound" "sli/plugins" "storage" "storage/esaas" "storage/pgaas" "utils")
+ccsdk_repos=("ccsdk" "ccsdk/dashboard" "ccsdk/distribution" "ccsdk/parent"
+"ccsdk/platform/blueprints" "ccsdk/platform/nbapi" "ccsdk/platform/plugins"
+"ccsdk/sli" "ccsdk/sli/adaptors" "ccsdk/sli/core" "ccsdk/sli/northbound"
+"ccsdk/sli/plugins" "ccsdk/storage" "ccsdk/storage/esaas" "ccsdk/storage/pgaas"
+"ccsdk/utils")
 
 # clone_ccsdk_repos() - Function that clones the CCSDK repositories
 function clone_ccsdk_repos {
-    clone_repo ccsdk $ccsdk_src_folder
-
-    for dirc in ${ccsdk_repos[@]}; do
-        clone_repo ccsdk/$dirc $ccsdk_src_folder/$dirc
+    for repo in ${ccsdk_repos[@]}; do
+        clone_repo $repo $ccsdk_src_folder${repo#ccsdk}
     done
 }
 
 # compile_ccsdk_repos() - 
 function compile_ccsdk_repos {
-    compile_src ccsdk $ccsdk_src_folder
-
-    for dirc in ${ccsdk_repos[@]}; do
-        compile_src $ccsdk_src_folder/$dirc
+    install_package unzip
+    for repo in ${ccsdk_repos[@]}; do
+        compile_src $ccsdk_src_folder${repo#ccsdk}
     done
 }
 
diff --git a/bootstrap/vagrant-onap/lib/dcae b/bootstrap/vagrant-onap/lib/dcae
index 4995c618d..78ca8de06 100755
--- a/bootstrap/vagrant-onap/lib/dcae
+++ b/bootstrap/vagrant-onap/lib/dcae
@@ -5,8 +5,12 @@ set -o xtrace
 source /var/onap/functions
 
 dcae_src_folder=$git_src_folder/dcae
-dcae_repos=("apod" "apod/analytics" "apod/buildtools" "apod/cdap" "collectors" "collectors/ves" "controller" "controller/analytics" "dcae-inventory"
-"demo" "demo/startup" "demo/startup/aaf" "demo/startup/controller" "demo/startup/message-router" "dmaapbc" "operation" "operation/utils" "pgaas" "utils" "utils/buildtools")
+dcae_repos=("dcae" "dcae/apod" "dcae/apod/analytics" "dcae/apod/buildtools"
+"dcae/apod/cdap" "dcae/collectors" "dcae/collectors/ves" "dcae/controller"
+"dcae/controller/analytics" "dcae/dcae-inventory" "dcae/demo"
+"dcae/demo/startup" "dcae/demo/startup/aaf" "dcae/demo/startup/controller"
+"dcae/demo/startup/message-router" "dcae/dmaapbc" "dcae/operation"
+"dcae/operation/utils" "dcae/orch-dispatcher" "dcae/pgaas" "dcae/utils" "dcae/utils/buildtools")
 
 # _create_config_file() - Creates a configuration yaml file for the controller
 function _create_config_file {
@@ -59,16 +63,15 @@ function _build_docker_image {
 
 # clone_all_dcae_repos() - Function that clones DCAE source repo.
 function clone_all_dcae_repos {
-    clone_repo dcae $dcae_src_folder
-    for dirc in ${dcae_repos[@]}; do
-        clone_repo dcae/$dirc $dcae_src_folder/$dirc
+    for repo in ${dcae_repos[@]}; do
+        clone_repo $repo $dcae_src_folder${repo#*dcae}
     done
 }
 
 # compile_all_dcae_repos() - Function that compiles DCAE source repo.
 function compile_all_dcae_repos {
-    for dirc in ${dcae_repos[@]}; do
-        compile_src $dcae_src_folder/$dirc
+    for repo in ${dcae_repos[@]}; do
+        compile_src $dcae_src_folder${repo#*dcae}
     done
 }
 
diff --git a/bootstrap/vagrant-onap/lib/mso b/bootstrap/vagrant-onap/lib/mso
index 6ff4bbe3e..5439d991b 100755
--- a/bootstrap/vagrant-onap/lib/mso
+++ b/bootstrap/vagrant-onap/lib/mso
@@ -5,23 +5,20 @@ set -o xtrace
 source /var/onap/functions
 
 src_folder=$git_src_folder/mso
-mso_repos=("chef-repo" "docker-config" "libs" "mso-config")
+mso_repos=("mso" "mso/chef-repo" "mso/docker-config" "mso/libs"
+"mso/mso-config")
 
 # clone_all_mso_repos() - Function that clones MSO source repo.
 function clone_all_mso_repos {
-    clone_repo mso $src_folder
-
-    for dirc in ${mso_repos[@]}; do
-        clone_repo mso/$dirc $src_folder/$dirc
+    for repo in ${mso_repos[@]}; do
+        clone_repo $repo $src_folder${repo#*mso}
     done
 }
 
 # compile_all_mso_repos() - Function that compiles MSO source repo.
 function compile_all_mso_repos {
-    compile_src $src_folder
-
-    for dirc in ${mso_repos[@]}; do
-        compile_src $src_folder/$dirc
+    for repo in ${mso_repos[@]}; do
+        compile_src $src_folder${repo#*mso}
     done
 }
 
diff --git a/bootstrap/vagrant-onap/lib/portal b/bootstrap/vagrant-onap/lib/portal
index 4dc5ef9b8..b5e768b2c 100755
--- a/bootstrap/vagrant-onap/lib/portal
+++ b/bootstrap/vagrant-onap/lib/portal
@@ -5,25 +5,29 @@ set -o xtrace
 source /var/onap/functions
 
 portal_src_folder=$git_src_folder/portal
-portal_repos=("sdk")
+portal_repos=("portal" "portal/sdk" "ecompsdkos" "ui/dmaapbc")
 
 # clone_all_portal_repos() - Function that clones Portal source repo.
 function clone_all_portal_repos {
-    clone_repo portal $portal_src_folder
-    clone_repo ecompsdkos $portal_src_folder/ecompsdkos
-    clone_repo "ui/dmaapbc" $portal_src_folder/dmaapbc
-
-    for dirc in ${portal_repos[@]}; do
-        clone_repo portal/$dirc $portal_src_folder/$dirc
+    for repo in ${portal_repos[@]}; do
+        if [[ "$repo" == "ui/dmaapbc" ]];then
+            prefix="ui"
+        else
+            prefix="portal"
+        fi
+        clone_repo $repo $portal_src_folder/${repo#*$prefix}
     done
 }
 
 # compile_all_portal_repos() - Function that compiles Portal source repo.
 function compile_all_portal_repos {
-    compile_src $portal_src_folder
-
-    for dirc in ${portal_repos[@]}; do
-        compile_src $portal_src_folder/$dirc
+    for repo in ${portal_repos[@]}; do
+        if [[ "$repo" == "ui/dmaapbc" ]];then
+            prefix="ui"
+        else
+            prefix="portal"
+        fi
+        compile_src $portal_src_folder/${repo#*$prefix}
     done
 }
 
diff --git a/bootstrap/vagrant-onap/lib/sdc b/bootstrap/vagrant-onap/lib/sdc
index 1d21e138b..a21e2a418 100755
--- a/bootstrap/vagrant-onap/lib/sdc
+++ b/bootstrap/vagrant-onap/lib/sdc
@@ -5,7 +5,9 @@ set -o xtrace
 source /var/onap/functions
 
 sdc_src_folder=$git_src_folder/sdc
-sdc_repos=("jtosca" "sdc-distribution-client" "sdc-docker-base" "sdc-titan-cassandra" "sdc-tosca" "sdc-vnfdesign" "sdc-workflow-designer" "sdc_common")
+sdc_repos=("sdc" "sdc/jtosca" "sdc/sdc-distribution-client"
+"sdc/sdc-docker-base" "sdc/sdc-titan-cassandra" "sdc/sdc-tosca"
+"sdc/sdc-vnfdesign" "sdc/sdc-workflow-designer" "sdc/sdc_common")
 
 # _mount_external_partition() - Create partition and mount the external volume
 function _mount_external_partition {
@@ -47,19 +49,15 @@ EOL
 
 # clone_all_sdc_repos() - Function that clones SDC source repo.
 function clone_all_sdc_repos {
-    clone_repo sdc $sdc_src_folder
-
-    for dirc in ${sdc_repos[@]}; do
-        clone_repo sdc/$dirc $sdc_src_folder/$dirc
+    for repo in ${sdc_repos[@]}; do
+        clone_repo $repo $sdc_src_folder${repo#*sdc}
     done
 }
 
 # compile_all_sdc_repos() - Function that compiles SDC source repo.
 function compile_all_sdc_repos {
-    compile_src sdc $sdc_src_folder
-
-    for dirc in ${sdc_repos[@]}; do
-        compile_src $sdc_src_folder/$dirc
+    for repo in ${sdc_repos[@]}; do
+        compile_src $sdc_src_folder${repo#*sdc}
     done
 }
 
diff --git a/bootstrap/vagrant-onap/lib/vid b/bootstrap/vagrant-onap/lib/vid
index 83b530995..f99fd6042 100755
--- a/bootstrap/vagrant-onap/lib/vid
+++ b/bootstrap/vagrant-onap/lib/vid
@@ -5,17 +5,20 @@ set -o xtrace
 source /var/onap/functions
 
 vid_src_folder=$git_src_folder/vid
+vid_repos=("vid" "vid/asdcclient")
 
 # clone_all_vid_repos() - Function that clones VID source code.
 function clone_all_vid_repos {
-    clone_repo vid $vid_src_folder
-    clone_repo vid/asdcclient $vid_src_folder/asdcclient
+    for repo in ${vid_repos[@]}; do
+        clone_repo $repo $vid_src_folder${repo#*vid}
+    done
 }
 
 # compile_all_vid_repos() - Function that compiles VID source repo.
 function compile_all_vid_repos {
-    compile_src $vid_src_folder/asdcclient
-    compile_src $vid_src_folder
+    for repo in ${vid_repos[@]}; do
+        compile_src $vid_src_folder${repo#*vid}
+    done
 }
 
 # _build_vid_images() - Function that builds VID docker images
diff --git a/bootstrap/vagrant-onap/tests/projects.txt b/bootstrap/vagrant-onap/tests/projects.txt
new file mode 100644
index 000000000..0aba508c9
--- /dev/null
+++ b/bootstrap/vagrant-onap/tests/projects.txt
@@ -0,0 +1,209 @@
+aaf/authz
+aaf/cadi
+aaf/inno
+aaf/luaplugin
+aai/aai-common
+aai/aai-config
+aai/aai-data
+aai/aai-service
+aai/babel
+aai/champ
+aai/data-router
+aai/esr-gui
+aai/esr-server
+aai/gizmo
+aai/logging-service
+aai/model-loader
+aai/resources
+aai/rest-client
+aai/router-core
+aai/search-data-service
+aai/sparky-be
+aai/sparky-fe
+aai/test-config
+aai/traversal
+appc
+appc/deployment
+ccsdk/dashboard
+ccsdk/distribution
+ccsdk/parent
+ccsdk/platform/blueprints
+ccsdk/platform/nbapi
+ccsdk/platform/plugins
+ccsdk/sli/adaptors
+ccsdk/sli/core
+ccsdk/sli/northbound
+ccsdk/sli/plugins
+ccsdk/storage/esaas
+ccsdk/storage/pgaas
+ccsdk/utils
+ci-management
+clamp
+cli
+dcae
+dcae/apod
+dcae/apod/analytics
+dcae/apod/buildtools
+dcae/apod/cdap
+dcae/collectors
+dcae/collectors/ves
+dcae/controller
+dcae/controller/analytics
+dcae/dcae-inventory
+dcae/demo
+dcae/demo/startup
+dcae/demo/startup/aaf
+dcae/demo/startup/controller
+dcae/demo/startup/message-router
+dcae/dmaapbc
+dcae/operation
+dcae/operation/utils
+dcae/orch-dispatcher
+dcae/pgaas
+dcae/utils
+dcae/utils/buildtools
+dcaegen2
+dcaegen2/analytics
+dcaegen2/analytics/tca
+dcaegen2/collectors
+dcaegen2/collectors/snmptrap
+dcaegen2/collectors/ves
+dcaegen2/deployments
+dcaegen2/platform
+dcaegen2/platform/blueprints
+dcaegen2/platform/cdapbroker
+dcaegen2/platform/cli
+dcaegen2/platform/configbinding
+dcaegen2/platform/deployment-handler
+dcaegen2/platform/inventory-api
+dcaegen2/platform/plugins
+dcaegen2/platform/policy-handler
+dcaegen2/platform/registrator
+dcaegen2/platform/servicechange-handler
+dcaegen2/utils
+demo
+dmaap/buscontroller
+dmaap/datarouter
+dmaap/dbcapi
+dmaap/messagerouter/dmaapclient
+dmaap/messagerouter/messageservice
+dmaap/messagerouter/mirroragent
+dmaap/messagerouter/msgrtr
+doc
+doc/tools
+ecompsdkos
+externalapi/nbi
+holmes/common
+holmes/dsa
+holmes/engine-management
+holmes/rule-management
+integration
+logging-analytics
+modeling/modelspec
+modeling/toscaparsers
+msb/apigateway
+msb/discovery
+msb/java-sdk
+msb/swagger-sdk
+mso
+mso/chef-repo
+mso/docker-config
+mso/libs
+mso/mso-config
+multicloud/azure
+multicloud/framework
+multicloud/openstack
+multicloud/openstack/vmware
+multicloud/openstack/windriver
+ncomp
+ncomp/cdap
+ncomp/core
+ncomp/docker
+ncomp/maven
+ncomp/openstack
+ncomp/sirius
+ncomp/sirius/manager
+ncomp/utils
+oom
+oom/registrator
+oparent
+optf/cmso
+optf/has
+optf/osdf
+policy/api
+policy/common
+policy/docker
+policy/drools-applications
+policy/drools-pdp
+policy/engine
+policy/gui
+policy/pap
+policy/pdp
+portal
+portal/sdk
+sdc
+sdc/jtosca
+sdc/sdc-distribution-client
+sdc/sdc-docker-base
+sdc/sdc-titan-cassandra
+sdc/sdc-tosca
+sdc/sdc-workflow-designer
+sdnc/adaptors
+sdnc/architecture
+sdnc/core
+sdnc/features
+sdnc/northbound
+sdnc/oam
+sdnc/parent
+sdnc/plugins
+so
+so/chef-repo
+so/docker-config
+so/libs
+so/so-config
+testsuite
+testsuite/heatbridge
+testsuite/properties
+testsuite/python-testing-utils
+ui
+ui/dmaapbc
+university
+usecase-ui
+usecase-ui/server
+vfc/gvnfm/vnflcm
+vfc/gvnfm/vnfmgr
+vfc/gvnfm/vnfres
+vfc/nfvo/catalog
+vfc/nfvo/driver/ems
+vfc/nfvo/driver/sfc
+vfc/nfvo/driver/vnfm/gvnfm
+vfc/nfvo/driver/vnfm/svnfm
+vfc/nfvo/lcm
+vfc/nfvo/resmanagement
+vfc/nfvo/wfengine
+vid
+vid/asdcclient
+vnfrqts/epics
+vnfrqts/guidelines
+vnfrqts/requirements
+vnfrqts/testcases
+vnfrqts/usecases
+vnfsdk/compliance
+vnfsdk/functest
+vnfsdk/lctest
+vnfsdk/model
+vnfsdk/pkgtools
+vnfsdk/refrepo
+vnfsdk/validation
+vvp/ansible-ice-bootstrap
+vvp/cms
+vvp/devkit
+vvp/documentation
+vvp/engagementmgr
+vvp/gitlab
+vvp/image-scanner
+vvp/jenkins
+vvp/portal
+vvp/postgresql
+vvp/test-engine
+vvp/validation-scripts
diff --git a/bootstrap/vagrant-onap/tests/test_ccsdk b/bootstrap/vagrant-onap/tests/test_ccsdk
index 2d2b87a54..81bdd4877 100644
--- a/bootstrap/vagrant-onap/tests/test_ccsdk
+++ b/bootstrap/vagrant-onap/tests/test_ccsdk
@@ -27,13 +27,13 @@ function test_clone_ccsdk_repos {
 }
 
 # test_compile_ccsdk_repos() - Verify the compilation of CCSDK repositories
-function compile_ccsdk_repos {
+function test_compile_ccsdk_repos {
     clone_ccsdk_repos
     compile_ccsdk_repos
 
-    asserts_file_exist $ccsdk_src_folder/dashboard/oom-app-common/target/oom-app-common-1.1.0-SNAPSHOT.jar
-    asserts_file_exist $ccsdk_src_folder/dashboard/oom-app-os/target/oom-app-os-1.1.0-SNAPSHOT.war
-    asserts_file_exist $ccsdk_src_folder/dashboard/oom-app-overlay/target/oom-app-overlay-1.1.0-SNAPSHOT.war
+    asserts_file_exist $ccsdk_src_folder/dashboard/ccsdk-app-common/target/ccsdk-app-common-1.1.0-SNAPSHOT.jar
+    asserts_file_exist $ccsdk_src_folder/dashboard/ccsdk-app-os/target/ccsdk-app-os-1.1.0-SNAPSHOT.war
+    asserts_file_exist $ccsdk_src_folder/dashboard/ccsdk-app-overlay/target/ccsdk-app-overlay-1.1.0-SNAPSHOT.war
     asserts_file_exist $ccsdk_src_folder/distribution/dgbuilder/target/dgbuilder.0.1.0-SNAPSHOT.zip
     asserts_file_exist $ccsdk_src_folder/distribution/platform-logic/installer/target/platform-logic-installer-0.1.0-SNAPSHOT.zip
     asserts_file_exist $ccsdk_src_folder/platform/nbapi/target/commonnbapi-0.0.3.war
diff --git a/bootstrap/vagrant-onap/tests/test_dcae b/bootstrap/vagrant-onap/tests/test_dcae
index 104cadd12..c6887cd78 100644
--- a/bootstrap/vagrant-onap/tests/test_dcae
+++ b/bootstrap/vagrant-onap/tests/test_dcae
@@ -28,6 +28,7 @@ function test_clone_all_dcae_repos {
     asserts_file_exist $dcae_src_folder/dmaapbc/pom.xml
     asserts_file_exist $dcae_src_folder/operation/README.md
     asserts_file_exist $dcae_src_folder/operation/utils/pom.xml
+    asserts_file_exist $dcae_src_folder/orch-dispatcher/LICENSE.txt
     asserts_file_exist $dcae_src_folder/pgaas/LICENSE.txt
     asserts_file_exist $dcae_src_folder/utils/README.md
     asserts_file_exist $dcae_src_folder/utils/buildtools/LICENSE.txt
diff --git a/bootstrap/vagrant-onap/tests/test_functions b/bootstrap/vagrant-onap/tests/test_functions
index ee7358a84..8043ebbd7 100644
--- a/bootstrap/vagrant-onap/tests/test_functions
+++ b/bootstrap/vagrant-onap/tests/test_functions
@@ -8,7 +8,7 @@ covered_functions=(
 "configure_bind" "install_java" "install_maven" "install_nodejs" "install_python"
 "install_docker" "pull_docker_image" "install_docker_compose" "configure_service"
 "start_ODL" "compile_src" "build_docker_image" "docker_openecomp_login"
-"pull_openecomp_image" "pull_onap_image"
+"pull_openecomp_image" "pull_onap_image" "coverity_repos"
 )
 
 # test_create_configuration_files() - Verify the creation of a configuration files
@@ -165,6 +165,32 @@ function test_build_docker_image {
     asserts_image onap/ccsdk-ubuntu-image
 }
 
+# test_coverity_repos() - Verify that all the repos are covered by scripts
+function test_coverity_repos {
+    # Get repositories list from source code files
+    repos_list=()
+    for module in $( stat -c "%F %n" /var/onap/* | grep -v "_\|functions\|commons\|files" | cut -d' ' -f 3-); do
+        source $module
+        module_repo_list=$(eval echo \${$(basename $module)_repos[@]})
+        repos_list=(${repos_list[@]} ${module_repo_list[@]})
+    done
+
+    pushd /var/onap_tests/
+    cp projects.txt remaining_projects.txt
+    for covered_repo in "${repos_list[@]}"; do
+        sed -i "/$covered_repo/d" remaining_projects.txt
+    done
+
+    threshold=75
+    num_projects=$(wc -l < projects.txt)
+    num_remaining_projects=$(wc -l < remaining_projects.txt)
+    coverage=`echo "scale=2; 100-($num_remaining_projects/$num_projects*100)" | bc | cut -d . -f 1`
+    if [ $coverage -lt $threshold ]; then
+        raise_error "There are repositories that are not covered by scripts"
+    fi
+    popd
+}
+
 if [ "$1" != '*' ]; then
     unset covered_functions
     covered_functions=$1
diff --git a/bootstrap/vagrant-onap/tools/update_project_list.sh b/bootstrap/vagrant-onap/tools/update_project_list.sh
new file mode 100755
index 000000000..2f0ebb9a3
--- /dev/null
+++ b/bootstrap/vagrant-onap/tools/update_project_list.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ssh $1@gerrit.onap.org -p 29418 gerrit ls-projects > projects.tmp
+tail -n +2 projects.tmp > tests/projects.txt
+rm projects.tmp