Update scripts to use python3

startODL scripts and certificate install scripts are updated

Issue-ID: CCSDK-2707
Change-Id: I67ab95c6d725b82fc381ca86a5be091e7aade803
Signed-off-by: Ravi Pendurty <ravi.pendurty@highstreet-technologies.com>

Former-commit-id: 031046c24f3dc81d0d04259deb35ed44382d1c39

4 files changed, 121 insertions, 32 deletions

diff --git a/installation/sdnc/src/main/scripts/installCerts.oom.py b/installation/sdnc/src/main/scripts/installCerts.oom.py
index ea76c672..42af7d2c 100644
--- a/installation/sdnc/src/main/scripts/installCerts.oom.py
+++ b/installation/sdnc/src/main/scripts/installCerts.oom.py
@@ -22,7 +22,7 @@
 
 # coding=utf-8
 import os
-import httplib
+import http.client
 import base64
 import time
 import zipfile
@@ -37,7 +37,7 @@ log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
 if not os.path.exists(log_directory):
     os.makedirs(log_directory)
 logging.basicConfig(filename=log_file,level=logging.DEBUG,filemode='w',format=log_format)
-print 'Start cert provisioning. Log file: ' + log_file;
+print ('Start cert provisioning. Log file: ' + log_file);
 
 Path = os.environ['ODL_CERT_DIR']
 
@@ -45,33 +45,36 @@ zipFileList = []
 
 username = os.environ['ODL_ADMIN_USERNAME']
 password = os.environ['ODL_ADMIN_PASSWORD']
+newpassword = os.environ.get('ODL_ADMIN_NEWPASSWORD')
 TIMEOUT=1000
 INTERVAL=30
 timePassed=0
 
-postKeystore= "/restconf/operations/netconf-keystore:add-keystore-entry"
-postPrivateKey= "/restconf/operations/netconf-keystore:add-private-key"
-postTrustedCertificate= "/restconf/operations/netconf-keystore:add-trusted-certificate"
+postKeystore= "/rests/operations/netconf-keystore:add-keystore-entry"
+postPrivateKey= "/rests/operations/netconf-keystore:add-private-key"
+postTrustedCertificate= "/rests/operations/netconf-keystore:add-trusted-certificate"
 
 envOdlFeaturesBoot='ODL_FEATURES_BOOT'
 # Strategy sli-api is default
 certreadyCmd="POST"
-certreadyUrl="/restconf/operations/SLI-API:healthcheck"
+certreadyUrl="/rests/operations/SLI-API:healthcheck"
 odlFeaturesBoot=os.environ.get(envOdlFeaturesBoot)
+
 if odlFeaturesBoot is not None:
     odlFeaturesBoot=odlFeaturesBoot.lower()
     if 'odl-netconf-topology' in odlFeaturesBoot or 'odl-netconf-clustered-topology' in odlFeaturesBoot:
         certreadyCmd="GET"
-        certreadyUrl="/restconf/operational/network-topology:network-topology"
+        certreadyUrl="/rests/data/network-topology:network-topology"
 logging.info('ODL ready strategy with command %s and url %s', certreadyCmd, certreadyUrl)
 
 cadi_file = '.pass'
 odl_port = 8181
-headers = {'Authorization':'Basic %s' % base64.b64encode(username + ":" + password),
+cred_string = username + ":" + password
+headers = {'Authorization':'Basic %s' %  base64.b64encode(cred_string.encode()).decode(),
            'X-FromAppId': 'csit-sdnc',
            'X-TransactionId': 'csit-sdnc',
            'Accept':"application/json",
-           'Content-type':"application/json"}
+           'Content-type':"application/yang-data+json"}
 
 def readFile(folder, file):
     key = open(Path + "/" + folder + "/" + file, "r")
@@ -107,8 +110,6 @@ def makeKeystoreKey(clientKey, count):
 
     return json_keystore_key
 
-
-
 def makePrivateKey(clientKey, clientCrt, certList, count):
     caPem = ""
     if certList:
@@ -175,7 +176,7 @@ def processFiles(folder, count):
 
 def post_content(clientKey, clientCrt, certList, count):
     logging.info('Post content: %d', count)
-    conn = httplib.HTTPConnection("localhost",odl_port)
+    conn = http.client.HTTPConnection("localhost",odl_port)
     if clientKey:
         json_keystore_key = makeKeystoreKey(clientKey, count)
         logging.debug("Posting private key in to ODL keystore")
@@ -197,7 +198,7 @@ def makeHealthcheckCall(headers, timePassed):
     # WAIT 10 minutes maximum and test every 30 seconds if HealthCheck API is returning 200
     while timePassed < TIMEOUT:
         try:
-            conn = httplib.HTTPConnection("localhost",odl_port)
+            conn = http.client.HTTPConnection("localhost",odl_port)
             req = conn.request(certreadyCmd, certreadyUrl,headers=headers)
             res = conn.getresponse()
             res.read()
@@ -292,11 +293,32 @@ def lookforfiles():
     else:
         logging.debug("No jks/p12 files found under cert directory %s", Path)
 
+def replaceAdminPassword(username, password, newpassword):
+    if newpassword is None:
+        logging.info('Not to replace password for user %s', username)
+    else:
+        logging.info('Replace password for user %s', username)
+        try:
+            jsondata = '{\"password\": \"{newpassword}\"}'.format(newpassword=newpassword)
+            url = '/auth/v1/users/{username}@sdn'.format(username=username)
+            loggin.info("Url %s data $s", url, jsondata)
+            conn = http.client.HTTPConnection("localhost",odl_port)
+            req = conn.request("PUT", url, jsondata, headers=headers)
+            res = conn.getresponse()
+            res.read()
+            httpStatus = res.status
+            if httpStatus == 200:
+                logging.debug("New password provided successfully for user %s", username)
+            else:
+                logging.debug("Password change was not possible. Problem code was: %d", httpStatus)
+        except:
+            logging.error("Cannot execute REST call to set password.")
 
 def readCertProperties():
     connected = makeHealthcheckCall(headers, timePassed)
     logging.info('Connected status: %s', connected)
     if connected:
+        replaceAdminPassword(username, password, newpassword)
         count = 0
         if os.path.isfile(Path + "/certs.properties"):
             with open(Path + "/certs.properties", "r") as f:
diff --git a/installation/sdnc/src/main/scripts/installCerts.py b/installation/sdnc/src/main/scripts/installCerts.py
index 354e6426..d00db393 100644
--- a/installation/sdnc/src/main/scripts/installCerts.py
+++ b/installation/sdnc/src/main/scripts/installCerts.py
@@ -21,7 +21,7 @@
 # coding=utf-8
 import os
 import re
-import httplib
+import http.client
 import base64
 import time
 import zipfile
@@ -47,9 +47,9 @@ TIMEOUT=1000
 INTERVAL=30
 timePassed=0
 
-postKeystore= "/restconf/operations/netconf-keystore:add-keystore-entry"
-postPrivateKey= "/restconf/operations/netconf-keystore:add-private-key"
-postTrustedCertificate= "/restconf/operations/netconf-keystore:add-trusted-certificate"
+postKeystore= "/rests/operations/netconf-keystore:add-keystore-entry"
+postPrivateKey= "/rests/operations/netconf-keystore:add-private-key"
+postTrustedCertificate= "/rests/operations/netconf-keystore:add-trusted-certificate"
 
 truststore_pass_file = Path + '/truststore.pass'
 truststore_file = Path + '/truststore.jks'
@@ -60,11 +60,12 @@ keystore_file = Path + '/keystore.jks'
 jks_files = [truststore_pass_file, keystore_pass_file, keystore_file, truststore_file]
 
 odl_port = 8181
-headers = {'Authorization':'Basic %s' % base64.b64encode(username + ":" + password),
+cred_string = username + ":" + password
+headers = {'Authorization':'Basic %s' % base64.b64encode(cred_string.encode()).decode(),
            'X-FromAppId': 'csit-sdnc',
            'X-TransactionId': 'csit-sdnc',
            'Accept':"application/json",
-           'Content-type':"application/json"}
+           'Content-type':"application/yang-data+json"}
 
 
 def readFile(folder, file):
@@ -172,7 +173,7 @@ def processFiles(folder, count):
 
 
 def post_content(clientKey, clientCrt, certList, count):
-    conn = httplib.HTTPConnection("localhost",odl_port)
+    conn = http.client.HTTPConnection("localhost",odl_port)
 
     if clientKey:
         json_keystore_key = makeKeystoreKey(clientKey, count)
@@ -195,8 +196,8 @@ def makeHealthcheckCall(headers, timePassed):
     # WAIT 10 minutes maximum and test every 30 seconds if HealthCheck API is returning 200
     while timePassed < TIMEOUT:
         try:
-            conn = httplib.HTTPConnection("localhost",odl_port)
-            req = conn.request("POST", "/restconf/operations/SLI-API:healthcheck",headers=headers)
+            conn = http.client.HTTPConnection("localhost",odl_port)
+            req = conn.request("POST", "/rests/operations/SLI-API:healthcheck",headers=headers)
             res = conn.getresponse()
             res.read()
             if res.status == 200:
@@ -332,4 +333,4 @@ def readCertProperties():
             process_jks_files(count)
 
 
-readCertProperties()
\ No newline at end of file
+readCertProperties()
diff --git a/installation/sdnc/src/main/scripts/startODL.oom.sh b/installation/sdnc/src/main/scripts/startODL.oom.sh
index f158c7dd..409e27c9 100755
--- a/installation/sdnc/src/main/scripts/startODL.oom.sh
+++ b/installation/sdnc/src/main/scripts/startODL.oom.sh
@@ -28,6 +28,9 @@
 
 ODL_HOME=${ODL_HOME:-/opt/opendaylight/current}
 ODL_FEATURES_BOOT_FILE=$ODL_HOME/etc/org.apache.karaf.features.cfg
+FEATURESBOOTMARKER="^featuresBoot *="
+REPOSITORIESBOOTMARKER="^featuresRepositories *="
+
 #
 ODL_REMOVEIDMDB=${ODL_REMOVEIDMDB:-false}
 
@@ -43,6 +46,7 @@ fi
 
 export ODL_ADMIN_PASSWORD ODL_ADMIN_USERNAME
 
+JDEBUG=${JDEBUG:-false}
 SDNC_HOME=${SDNC_HOME:-/opt/onap/sdnc}
 SDNC_BIN=${SDNC_BIN:-/opt/onap/sdnc/bin}
 CCSDK_HOME=${CCSDK_HOME:-/opt/onap/ccsdk}
@@ -65,6 +69,7 @@ SDNRWT=${SDNRWT:-false}
 SDNRWT_BOOTFEATURES=${SDNRWT_BOOTFEATURES:-sdnr-wt-feature-aggregator}
 SDNRDM=${SDNRDM:-false}
 # Add devicemanager base and specific repositories
+SDNR_BASE_REPO=${SDNRDM_BASE_REPO:-mvn:org.onap.ccsdk.features.sdnr.wt/sdnr-wt-feature-aggregator/$CCSDKFEATUREVERSION/xml/features}
 SDNRDM_BASE_REPO=${SDNRDM_BASE_REPO:-mvn:org.onap.ccsdk.features.sdnr.wt/sdnr-wt-feature-aggregator-devicemanager-base/$CCSDKFEATUREVERSION/xml/features}
 SDNRDM_ONF_REPO=${SDNRDM_ONF_REPO:-mvn:org.onap.ccsdk.features.sdnr.wt/sdnr-wt-devicemanager-onf-feature/$CCSDKFEATUREVERSION/xml/features}
 SDNRDM_ORAN_REPO=${SDNRDM_ORAN_REPO:-mvn:org.onap.ccsdk.features.sdnr.wt/sdnr-wt-devicemanager-oran-feature/$CCSDKFEATUREVERSION/xml/features}
@@ -83,6 +88,8 @@ SDNRDBCOMMAND=${SDNRDBCOMMAND:--c init -db $SDNRDBURL -dbu $SDNRDBUSERNAME -dbp
 SDNR_NORTHBOUND=${SDNR_NORTHBOUND:-false}
 SDNR_NORTHBOUND_BOOTFEATURES=${SDNR_NORTHBOUND_BOOTFEATURES:-sdnr-northbound-all}
 
+#OVERRIDE_FEATURES_BOOT
+
 # Functions
 
 # Test if repository exists, like this mvn:org.onap.ccsdk.features.sdnr.wt/sdnr-wt-devicemanager-oran-feature/0.7.2/xml/features
@@ -108,6 +115,19 @@ function addRepository() {
     echo "Repo does not exist: $1"
   fi
 }
+# Append features to karaf boot feature configuration
+# $1 search pattern .. no leading ","
+# $2 replacement .. if "" remove
+function replaceRepository() {
+  CFG=$ODL_FEATURES_BOOT_FILE
+  if [ -n "$2" ] ; then
+    echo "Replace feature repo $1 with: $2"
+    sed -i "/$REPOSITORIESBOOTMARKER/ s/,* *$1/,$2/g" $CFG
+  else
+    echo "Remove feature repo $1"
+    sed -i "/$REPOSITORIESBOOTMARKER/ s/,* *$1//g" $CFG
+  fi
+}
 
 # Append features to karaf boot feature configuration
 # $1 additional feature to be added
@@ -119,22 +139,28 @@ function addToFeatureBoot() {
     addRepository $2
   fi
   echo "Add boot feature: $1"
-  sed -i "\|featuresBoot *=|s|$|,$1|" $CFG
+  sed -i "\|$FEATURESBOOTMARKER|s|$|,$1|" $CFG
 }
 
 # Append features to karaf boot feature configuration
-# $1 search pattern
-# $2 replacement
+# $1 search pattern .. no leading ","
+# $2 replacement .. if "" remove
 function replaceFeatureBoot() {
   CFG=$ODL_FEATURES_BOOT_FILE
-  echo "Replace boot feature $1 with: $2"
-  sed -i "/featuresBoot/ s/$1/$2/g" $CFG
+  if [ -n "$2" ] ; then
+    echo "Replace boot feature $1 with: $2"
+    sed -i "/$FEATURESBOOTMARKER/ s/,* *$1/,$2/g" $CFG
+  else
+    echo "Remove boot feature $1"
+    sed -i "/$FEATURESBOOTMARKER/ s/,* *$1//g" $CFG
+  fi
 }
 
 # Remove all sdnc specific features
 function cleanupFeatureBoot() {
   echo "Remove northbound bootfeatures "
-  sed -i "/featuresBoot/ s/,ccsdk-sli-core-all.*$//g" $ODL_FEATURES_BOOT_FILE
+  sed -i "/$FEATURESBOOTMARKER/ s/,ccsdk-sli-core-all.*$//g" $ODL_FEATURES_BOOT_FILE
+  sed -i "/$FEATURESBOOTMARKER/ s/odl-restconf-nb-rfc8040,//g" $ODL_FEATURES_BOOT_FILE
 }
 
 function initialize_sdnr() {
@@ -155,6 +181,19 @@ function initialize_sdnr() {
 function install_sdnrwt_features() {
   # Repository setup provided via sdnc dockerfile
   if $SDNRWT; then
+    #Clean up <-----------------
+    #Uses wrong version
+    echo "Remove sdnr-northbound-all and add BGP"
+    replaceFeatureBoot "sdnr-northbound-all"
+    #Add missing org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.bgp.types.rev180329
+    #addRepository "mvn:org.opendaylight.bgpcep/odl-bgpcep-bgp-dependencies/0.11.1/xml/features"
+    #replaceFeatureBoot "odl-daexim-all" "odl-daexim-all, odl-bgpcep-bgp-dependencies"
+
+    # remove old existing
+    replaceRepository "mvn:org.onap.ccsdk.features.sdnr.wt\/sdnr-wt-feature-aggregator\/[^\/]*\/xml\/features"
+    replaceRepository "mvn:org.onap.ccsdk.features.sdnr.wt\/sdnr-wt-feature-aggregator-devicemanager\/[^\/]*\/xml\/features"
+    # Add devicemanagers
+    addRepository $SDNR_BASE_REPO
     addRepository $SDNRDM_BASE_REPO
     addRepository $SDNRDM_ONF_REPO
     addRepository $SDNRDM_ORAN_REPO
@@ -162,6 +201,7 @@ function install_sdnrwt_features() {
 
     if $SDNRONLY; then
       cleanupFeatureBoot
+      addToFeatureBoot ccsdk-aafshiro
     fi
     if $SDNRDM; then
       addToFeatureBoot "$SDNRDM_BOOTFEATURES"
@@ -240,14 +280,31 @@ function enable_odl_cluster(){
 # -----------------------
 # Main script starts here
 
+if $JDEBUG ; then
+    echo "Activate remote debugging"
+    #JSTADTPOLICYFILE="$ODL_HOME/etc/tools.policy"
+    #echo -e "grant codebase \"file:${JAVA_HOME}/lib/tools.jar\" {\n  permission java.security.AllPermission;\n };" > $JSTADTPOLICYFILE
+    #sleep 1
+    #$JAVA_HOME/bin/jstatd -p 1089 -J-Djava.security.policy=$JSTADTPOLICYFILE &
+    EXTRA_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.port=1090"
+    EXTRA_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.rmi.port=1090"
+    EXTRA_JAVA_OPTS+=" -Djava.rmi.server.hostname=$HOSTNAME"
+    EXTRA_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.local.only=false"
+    EXTRA_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.ssl=false"
+    EXTRA_JAVA_OPTS+=" -Dcom.sun.management.jmxremote.authenticate=false"
+    export EXTRA_JAVA_OPTS
+fi
+
 echo "Image path=${IMAGEPATH}"
 echo "Image names=${IMAGENAMES}"
+echo "Hostname=${HOSTNAME}"
 echo "Settings:"
 echo "  USER=$(whoami)"
 echo "  SDNC_BIN=$SDNC_BIN"
 echo "  SDNC_HOME=$SDNC_HOME"
 echo "  ODL_CERT_DIR=$ODL_CERT_DIR"
 echo "  CCSDKFEATUREVERSION=$CCSDKFEATUREVERSION"
+echo "  OVERRIDE_FEATURES_BOOT=$OVERRIDE_FEATURES_BOOT"
 echo "  ENABLE_ODL_CLUSTER=$ENABLE_ODL_CLUSTER"
 echo "  ODL_REMOVEIDMDB=$ODL_REMOVEIDMDB"
 echo "  SDNC_REPLICAS=$SDNC_REPLICAS"
@@ -263,6 +320,8 @@ echo "  IS_PRIMARY_CLUSTER=$IS_PRIMARY_CLUSTER"
 echo "  MY_ODL_CLUSTER=$MY_ODL_CLUSTER"
 echo "  PEER_ODL_CLUSTER=$PEER_ODL_CLUSTER"
 echo "  AAF_ENABLED=$SDNC_AAF_ENABLED"
+echo "  JDEBUG=$JDEBUG"
+echo "  EXTRA_JAVA_OPTS=$EXTRA_JAVA_OPTS"
 
 if $SDNC_AAF_ENABLED; then
     export SDNC_STORE_DIR=/opt/app/osaaf/local
@@ -306,17 +365,24 @@ then
     echo "Installed at `date`" > ${SDNC_HOME}/.installed
 fi
 
+if [ -n "$OVERRIDE_FEATURES_BOOT" ] ; then
+  echo "Override features boot: $OVERRIDE_FEATURES_BOOT"
+  sed -i "/$FEATURESBOOTMARKER/c\featuresBoot = $OVERRIDE_FEATURES_BOOT" $ODL_FEATURES_BOOT_FILE
+fi
+
 # Odl configuration done
-ODL_FEATURES_BOOT=$(sed -n "/featuresBoot =/p" $ODL_FEATURES_BOOT_FILE)
+ODL_REPOSITORIES_BOOT=$(sed -n "/$REPOSITORIESBOOTMARKER/p" $ODL_FEATURES_BOOT_FILE)
+ODL_FEATURES_BOOT=$(sed -n "/$FEATURESBOOTMARKER/p" $ODL_FEATURES_BOOT_FILE)
 export ODL_FEATURES_BOOT
 
 if [ -z "$ODL_CERT_DIR" ] ; then
   echo "No certs provided. Skip installation."
 else
   echo "Start background cert installer"
-  nohup python ${SDNC_BIN}/installCerts.oom.py &
+  nohup python3 ${SDNC_BIN}/installCerts.oom.py &
 fi
 
 echo "Startup opendaylight"
+echo $ODL_REPOSITORIES_BOOT
 echo $ODL_FEATURES_BOOT
 exec ${ODL_HOME}/bin/karaf server
diff --git a/installation/sdnc/src/main/scripts/startODL.sh b/installation/sdnc/src/main/scripts/startODL.sh
index 6f9bdad2..14ffe2a3 100755
--- a/installation/sdnc/src/main/scripts/startODL.sh
+++ b/installation/sdnc/src/main/scripts/startODL.sh
@@ -176,6 +176,6 @@ cp /var/custom-certs/* /tmp
 # Create ODL data log directory (it nornally is created after karaf
 # is started, but needs to exist before installCerts.py runs)
 mkdir -p /opt/opendaylight/data/log
-nohup python ${SDNC_BIN}/installCerts.py &
+nohup python3 ${SDNC_BIN}/installCerts.py &
 
 exec ${ODL_HOME}/bin/karaf server