Use python3 for CSITs

- use python3 and pip3 in scripts
- update robot-framework-http library, so it can use python3
- update selenium2library, so it can use python3

Fixed merge conflict from cherry pick:
- prepare-csit.sh

Changed to github instead of gerrit-mirror, as the mirror is flakey and
broke one of the CSIT verify jobs.

Issue-ID: POLICY-3092
Signed-off-by: arkadiusz.adamski <aadamski@est.tech>
Change-Id: I5a6a4fe43e54366859049e1d43bbce599dacd14e
(cherry picked from commit 2b1e8d08ed869124e4a363a0c5f58999f487a4be)
Signed-off-by: Jim Hahn <jrh3@att.com>

12 files changed, 96 insertions, 37 deletions

diff --git a/csit/apex-pdp/plans/setup.sh b/csit/apex-pdp/plans/setup.sh
index ee8709d4..89f780a0 100644
--- a/csit/apex-pdp/plans/setup.sh
+++ b/csit/apex-pdp/plans/setup.sh
@@ -4,6 +4,7 @@
 #
 #  Modifications copyright (c) 2019 Nordix Foundation.
 #  Modifications Copyright (C) 2020-2021 AT&T Intellectual Property.
+#  Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,9 +23,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 
diff --git a/csit/api/plans/setup.sh b/csit/api/plans/setup.sh
index 65d71e93..28d08b75 100644
--- a/csit/api/plans/setup.sh
+++ b/csit/api/plans/setup.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 #  Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
+# Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,9 +22,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 
 sudo apt-get -y install libxml2-utils
diff --git a/csit/distribution/plans/setup.sh b/csit/distribution/plans/setup.sh
index 664cb69f..54ee68f1 100644
--- a/csit/distribution/plans/setup.sh
+++ b/csit/distribution/plans/setup.sh
@@ -4,6 +4,7 @@
 #
 #  Modifications copyright (c) 2019 Nordix Foundation.
 #  Modifications Copyright (C) 2020-2021 AT&T Intellectual Property.
+#  Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,9 +23,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 
diff --git a/csit/drools-applications/plans/setup.sh b/csit/drools-applications/plans/setup.sh
index 02d854c7..f8ca6326 100755
--- a/csit/drools-applications/plans/setup.sh
+++ b/csit/drools-applications/plans/setup.sh
@@ -2,6 +2,7 @@
 #
 # ===========LICENSE_START====================================================
 #  Copyright (C) 2019-2021 AT&T Intellectual Property. All rights reserved.
+#  Modification Copyright 2021. Nordix Foundation.
 # ============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,9 +20,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 
diff --git a/csit/drools-pdp/plans/setup.sh b/csit/drools-pdp/plans/setup.sh
index 003f21ba..3f3624fa 100755
--- a/csit/drools-pdp/plans/setup.sh
+++ b/csit/drools-pdp/plans/setup.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 # Copyright 2017-2021 AT&T Intellectual Property. All rights reserved.
+#  Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,9 +22,9 @@ source ${SCRIPTS}/get-branch-mariadb.sh
 # OS upgrades
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 
diff --git a/csit/get-models-examples.sh b/csit/get-models-examples.sh
index 75b65288..e83b0245 100644
--- a/csit/get-models-examples.sh
+++ b/csit/get-models-examples.sh
@@ -24,4 +24,4 @@ mkdir ${WORKSPACE}/models
 cd ${WORKSPACE}
 
 # download models examples
-git clone -b ${GERRIT_BRANCH} --single-branch git://gerrit-mirror-ap.onap.org/mirror/policy/models.git
+git clone -b ${GERRIT_BRANCH} --single-branch https://github.com/onap/policy-models.git models
diff --git a/csit/include-raw-integration-install-robotframework.sh b/csit/include-raw-integration-install-robotframework.sh
new file mode 100644
index 00000000..65b838b1
--- /dev/null
+++ b/csit/include-raw-integration-install-robotframework.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2015 The Linux Foundation and others.
+# Modification Copyright 2021. Nordix Foundation.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+# vim: sw=4 ts=4 sts=4 et ft=sh :
+
+ROBOT_VENV="/tmp/v/robot"
+echo ROBOT_VENV="${ROBOT_VENV}" >> "${WORKSPACE}/env.properties"
+
+echo "Python version is: $(python3 --version)"
+
+# The --system-site-packages parameter allows us to pick up system level
+# installed packages. This allows us to bake matplotlib which takes very long
+# to install into the image.
+python3 -m venv "${ROBOT_VENV}"
+
+source "${ROBOT_VENV}/bin/activate"
+
+set -exu
+
+# Make sure pip3 itself us up-to-date.
+python3 -m pip3 install --upgrade pip3
+
+echo "Installing Python Requirements"
+python3 -m pip3 install -r pylibs.txt
+odltools -V
+pip3 freeze
diff --git a/csit/pap/plans/setup.sh b/csit/pap/plans/setup.sh
index d53edc1c..97cdb307 100644
--- a/csit/pap/plans/setup.sh
+++ b/csit/pap/plans/setup.sh
@@ -2,6 +2,7 @@
 # ============LICENSE_START=======================================================
 #  Copyright (C) 2019 Nordix Foundation.
 #  Modifications Copyright (C) 2019-2021 AT&T Intellectual Property.
+#  Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,9 +22,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 bash ${SCRIPTS}/get-models-examples.sh
diff --git a/csit/prepare-csit.sh b/csit/prepare-csit.sh
index 9df2207b..4297696f 100755
--- a/csit/prepare-csit.sh
+++ b/csit/prepare-csit.sh
@@ -2,6 +2,7 @@
 #
 # Copyright 2019 © Samsung Electronics Co., Ltd.
 # Modification Copyright 2021 © AT&T Intellectual Property.
+# Modification Copyright 2021. Nordix Foundation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,25 +36,14 @@ else
     rm -rf /tmp/ci-management
     rm -f ${WORKSPACE}/env.properties
     cd /tmp
-    git clone -b master --single-branch git://gerrit-mirror-ap.onap.org/mirror/ci-management.git
+    git clone -b master --single-branch https://github.com/onap/ci-management.git
     source /tmp/ci-management/jjb/integration/include-raw-integration-install-robotframework.sh
 fi
 
-# install required Robot libraries
-pip install robotframework-selenium2library==1.8.0 robotframework-extendedselenium2library==0.9.1
 
 # install eteutils
 mkdir -p ${ROBOT_VENV}/src/onap
 rm -rf ${ROBOT_VENV}/src/onap/testsuite
-pip install --upgrade --extra-index-url="https://nexus3.onap.org/repository/PyPi.staging/simple" 'robotframework-onap==0.5.1.*' --pre
+pip3 install --upgrade --extra-index-url="https://nexus3.onap.org/repository/PyPi.staging/simple" 'robotframework-onap==0.5.1.*' --pre
 
-pip freeze
-
-# install chrome driver
-if [ ! -x ${ROBOT_VENV}/bin/chromedriver ]; then
-    pushd ${ROBOT_VENV}/bin
-    wget -N http://chromedriver.storage.googleapis.com/2.35/chromedriver_linux64.zip
-    unzip chromedriver_linux64.zip
-    chmod +x chromedriver
-    popd
-fi
+pip3 freeze
diff --git a/csit/pylibs.txt b/csit/pylibs.txt
new file mode 100644
index 00000000..d6250dba
--- /dev/null
+++ b/csit/pylibs.txt
@@ -0,0 +1,27 @@
+docker-py
+ipaddr
+netaddr
+netifaces
+pyhocon
+requests
+robotframework-httplibrary
+robotframework-requests
+robotframework-selenium2library
+robotframework-extendedselenium2library
+robotframework-sshlibrary
+scapy
+# Module jsonpath is needed by current AAA idmlite suite.
+jsonpath-rw
+# Modules for longevity framework robot library
+elasticsearch
+elasticsearch-dsl
+# Module for pyangbind used by lispflowmapping project
+pyangbind
+# Module for iso8601 datetime format
+isodate
+# Module for TemplatedRequests.robot library
+jmespath
+# Module for backup-restore support library
+jsonpatch
+# odltools for extra debugging
+odltools
diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh
index 8941a0fa..3b4d0764 100755
--- a/csit/run-project-csit.sh
+++ b/csit/run-project-csit.sh
@@ -3,6 +3,7 @@
 # Copyright 2016-2017 Huawei Technologies Co., Ltd.
 # Modification Copyright 2019 © Samsung Electronics Co., Ltd.
 # Modification Copyright 2021 © AT&T Intellectual Property.
+# Modification Copyright 2021. Nordix Foundation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -187,7 +188,7 @@ SUITES=$( xargs -a testplan.txt )
 echo ROBOT_VARIABLES="${ROBOT_VARIABLES}"
 echo "Starting Robot test suites ${SUITES} ..."
 relax_set
-python -m robot.run -N ${PROJECT} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${SUITES}
+python3 -m robot.run -N ${PROJECT} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${SUITES}
 RESULT=$?
 load_set
 echo "RESULT: ${RESULT}"
diff --git a/csit/xacml-pdp/plans/setup.sh b/csit/xacml-pdp/plans/setup.sh
index 21d88213..cf369ab9 100644
--- a/csit/xacml-pdp/plans/setup.sh
+++ b/csit/xacml-pdp/plans/setup.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 #  Copyright (C) 2020-2021 AT&T Intellectual Property. All rights reserved.
+#  Modification Copyright 2021. Nordix Foundation.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,9 +20,9 @@
 source ${SCRIPTS}/get-branch-mariadb.sh
 
 echo "Uninstall docker-py and reinstall docker."
-pip uninstall -y docker-py
-pip uninstall -y docker
-pip install -U docker==2.7.0
+pip3 uninstall -y docker-py
+pip3 uninstall -y docker
+pip3 install -U docker
 
 sudo apt-get -y install libxml2-utils
 bash ${SCRIPTS}/get-models-examples.sh