Refactoring the sdc-BE-init python scripts

Deeper refactoring of python script:
- create the python package with init script
- support for python 3.x
- reuse code
- new design
- support for .json conf file
- update the docker chef script

Issue-ID: SDC-2784
Signed-off-by: Krystian Kedron <k.kedron@partner.samsung.com>
Change-Id: I02169eb7d0e3e90851ba1811536d1712c3b4145f

7 files changed, 88 insertions, 176 deletions

diff --git a/catalog-be/sdc-backend-init/Dockerfile b/catalog-be/sdc-backend-init/Dockerfile
index cb26610859..3ffcbe691a 100644
--- a/catalog-be/sdc-backend-init/Dockerfile
+++ b/catalog-be/sdc-backend-init/Dockerfile
@@ -1,11 +1,22 @@
-FROM onap/base_sdc-python:1.6.0-SNAPSHOT-latest
+FROM onap/base_sdc-python:1.7.0
 
-COPY --chown=sdc:sdc chef-solo /home/sdc/chef-solo/
+# user/group args are inherited from base_sdc-python base image
+COPY --chown=$user:$group scripts /home/$user/scripts
 
-COPY --chown=sdc:sdc chef-repo/cookbooks /home/sdc/chef-solo/cookbooks/
+RUN chmod 770 -R /home/$user/scripts && \
+    cd /home/$user/scripts && \
+    python setup.py install --user
 
-COPY --chown=sdc:sdc startup.sh /home/sdc/
+ENV PATH=$PATH:/home/$user/.local/bin
 
-RUN chmod 770 /home/sdc/startup.sh
+COPY --chown=$user:$group chef-solo /home/$user/chef-solo/
 
-ENTRYPOINT [ "/home/sdc/startup.sh" ]
+COPY --chown=$user:$group chef-repo/cookbooks /home/$user/chef-solo/cookbooks/
+
+COPY --chown=$user:$group startup.sh /home/$user/
+
+RUN chmod 770 /home/$user/startup.sh
+
+WORKDIR /home/$user/
+
+ENTRYPOINT /home/${user}/startup.sh
diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/1_create_consumer_and_user.rb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/1_create_consumer_and_user.rb
index 97d140ab1f..bf8065b079 100644
--- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/1_create_consumer_and_user.rb
+++ b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/1_create_consumer_and_user.rb
@@ -1,46 +1,43 @@
 if node['disableHttp']
   protocol = "https"
+  https_flag = "--https"
   be_port = node['BE']['https_port']
 else
   protocol = "http"
+  https_flag = ""
   be_port = node['BE']['http_port']
 end
 
-
-template "/var/tmp/user.py" do
-    source "user.py.erb"
-    sensitive true
-    mode 0755
-    variables({
-      :protocol => protocol,
-      :be_ip => node['Nodes']['BE'],
-      :be_port => be_port
-    })
+if node['BE']['user_conf']
+  user_conf_dir = "--conf " + node['BE']['user_conf']
+else
+  # set default user configuration file
+  user_conf_dir = ""
 end
 
 bash "executing-create_user" do
-   code <<-EOH
-     python /var/tmp/user.py
-     rc=$?
-     if [[ $rc != 0 ]]; then exit $rc; fi
-   EOH
+  code <<-EOH
+    sdcuserinit -i #{node['Nodes']['BE']} -p #{be_port} #{user_conf_dir} #{https_flag}
+    rc=$?
+    if [[ $rc != 0 ]]; then exit $rc; fi
+  EOH
 end
 
 template "/var/tmp/consumers.py" do
-    source "consumers.py.erb"
-    sensitive true
-    mode 0755
-    variables({
-      :protocol => protocol,
-      :be_ip => node['Nodes']['BE'],
-      :be_port => be_port
-    })
+  source "consumers.py.erb"
+  sensitive true
+  mode 0755
+  variables({
+    :protocol => protocol,
+    :be_ip => node['Nodes']['BE'],
+    :be_port => be_port
+  })
 end
 
 bash "executing-consumers" do
-   code <<-EOH
-     python /var/tmp/consumers.py
-     rc=$?
-     if [[ $rc != 0 ]]; then exit $rc; fi
-   EOH
+  code <<-EOH
+    python /var/tmp/consumers.py
+    rc=$?
+    if [[ $rc != 0 ]]; then exit $rc; fi
+  EOH
 end
diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb
index b640b3afe9..f59d91693d 100644
--- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb
+++ b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb
@@ -3,7 +3,7 @@ be_ip=node['Nodes'][:BE]
 if node['disableHttp']
   protocol = "https"
   be_port = node['BE']['https_port']
-  param="-i #{be_ip} -p #{be_port} --scheme #{protocol}"
+  param="-i #{be_ip} -p #{be_port} --https"
 else
   protocol = "http"
   be_port = node['BE']['http_port']
@@ -11,38 +11,39 @@ else
 end
 
 cookbook_file "/var/tmp/normatives.tar.gz" do
-      source "normatives.tar.gz"
-   end
+  source "normatives.tar.gz"
+end
 
 bash "executing-import_Normatives" do
-   code <<-EOH
-     set -x
-     cd /var/tmp
-     tar xvfz /var/tmp/normatives.tar.gz
-     cd /var/tmp/normatives/scripts/import/tosca/
-     /bin/chmod +x *.py
-
-     # executing the normatives
-     # add --debug=true to the importNormativeAll.py arguments to enable debug
-
-     check_normative="/tmp/check_normative.out"
-     status_code=$(curl -k -s -o ${check_normative} -w "%{http_code}\\n" -X GET -H 'Content-Type: application/json;charset=UTF-8' -H 'USER_ID: jh0003' -H 'X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090' "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen")
-     if [ "$status_code" != 200 ] ; then
-        exit "$status_code"
-     fi
+  code <<-EOH
+    set -ex
 
-     #curl -s -X GET -H "Content-Type: application/json;charset=UTF-8" -H "USER_ID: jh0003" -H "X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090" "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen" > ${check_normative}
+    cd /var/tmp/
+    tar -xvf normatives.tar.gz
+    
+    # executing the normatives
+    # add --debug to the importNormativeAll.py arguments to enable debug
+    
+    check_normative="/tmp/check_normative.out"
+    status_code=$(curl -k -s -o ${check_normative} -w "%{http_code}\\n" -X GET -H 'Content-Type: application/json;charset=UTF-8' -H 'USER_ID: jh0003' -H 'X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090' "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen")
+    if [ "$status_code" != 200 ] ; then
+      exit "$status_code"
+    fi
+    
+    #curl -s -X GET -H "Content-Type: application/json;charset=UTF-8" -H "USER_ID: jh0003" -H "X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090" "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen" > ${check_normative}
+    
+    resources_len=`cat ${check_normative}| jq '.["resources"]|length'`
+    mkdir -p /var/lib/jetty/logs
 
-     resources_len=`cat ${check_normative}| jq '.["resources"]|length'`
-     mkdir -p /var/lib/jetty/logs
-     if [ $resources_len -eq 0 ] ; then
-        python importONAPNormativeAll.py #{param} > /var/lib/jetty/logs/importNormativeAll.log
-            rc=$?
-            if [[ $rc != 0 ]]; then exit $rc; fi
-     else
-        python upgradeONAPNormative.py #{param} > /var/lib/jetty/logs/upgradeNormative.log
-            rc=$?
-            if [[ $rc != 0 ]]; then exit $rc; fi
-     fi
-   EOH
+    cd /var/tmp/normatives/import/tosca/
+    if [ $resources_len -eq 0 ] ; then
+      sdcimportall #{param} > /var/lib/jetty/logs/importNormativeAll.log
+      rc=$?
+      if [[ $rc != 0 ]]; then exit $rc; fi
+    else
+      sdcupgradeall #{param} > /var/lib/jetty/logs/upgradeNormative.log
+      rc=$?
+      if [[ $rc != 0 ]]; then exit $rc; fi
+    fi
+  EOH
 end
diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/check_Backend_Health.py.erb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/check_Backend_Health.py.erb
index bcacf888ed..ebd8ea5c84 100644
--- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/check_Backend_Health.py.erb
+++ b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/check_Backend_Health.py.erb
@@ -30,7 +30,7 @@ def checkBackend():
     proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE )
     (out, err) = proc.communicate()
     result = out.strip()
-    return result
+    return str(result, "UTF-8")
 
 ##############################
 #    Main
@@ -39,12 +39,12 @@ def checkBackend():
 for i in range(1,10):
     myResult = checkBackend()
     if myResult == '200':
-        print '[INFO]: Backend is up and running'
+        print('[INFO]: Backend is up and running')
         beStat=1
         break
     else:
         currentTime = datetime.now()
-        print '[ERROR]: ' + currentTime.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + ' Backend not responding, try #' + str(i) + bcolors.ENDC
+        print('[ERROR]: ' + currentTime.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + ' Backend not responding, try #' + str(i) + bcolors.ENDC)
         time.sleep(10)
 
 if beStat == 0:
diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/consumers.py.erb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/consumers.py.erb
index b09b71948a..02ca9a8109 100644
--- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/consumers.py.erb
+++ b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/consumers.py.erb
@@ -28,7 +28,7 @@ def checkBackend():
     proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE )
     (out, err) = proc.communicate()
     result = out.strip()
-    return result
+    return str(result, "UTF-8")
 
 
 def checkConsumer(consumerName):
@@ -37,18 +37,18 @@ def checkConsumer(consumerName):
     proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE )
     (out, err) = proc.communicate()
     result = out.strip()
-    return result
+    return str(result, "UTF-8")
 
 
 def createConsumer( consumerName, consumerSalt, consumerPass ):
-    print '[INFO] ' + consumerName
+    print('[INFO] ' + consumerName)
     command="curl -k -s -o /dev/null -w \"%{http_code}\" -X POST -i -H \"Accept: application/json; charset=UTF-8\" -H \"Content-Type: application/json\" -H \"USER_ID: jh0003\" "+PROTOCOL+"://"  + BE_IP  + ":" + BE_PORT + "/sdc2/rest/v1/consumers/ -d '{\"consumerName\": '" + consumerName + "', \"consumerSalt\": '" + consumerSalt + "',\"consumerPassword\": '" + consumerPass + "'}'"
 
     proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE)
 
     (out, err) = proc.communicate()
     result = out.strip()
-    return result
+    return str(result, "UTF-8")
 
 
 
@@ -69,25 +69,25 @@ beStat=0
 for i in range(1,10):
     myResult = checkBackend()
     if myResult == '200':
-        print '[INFO]: Backend is up and running'
+        print('[INFO]: Backend is up and running')
         beStat=1
         break
     else:
         currentTime = datetime.now()
-        print '[ERROR]: ' + currentTime.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + ' Backend not responding, try #' + str(i) + bcolors.ENDC
+        print('[ERROR]: ' + currentTime.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + ' Backend not responding, try #' + str(i) + bcolors.ENDC)
         time.sleep(10)
 
 if beStat == 0:
-    print '[ERROR]: ' + time.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + 'Backend is DOWN :-(' + bcolors.ENDC
+    print('[ERROR]: ' + time.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + 'Backend is DOWN :-(' + bcolors.ENDC)
     exit()
 
 for consumer in consumersList:
     myResult = checkConsumer(consumer)
     if myResult == '200':
-        print '[INFO]: ' + consumer + ' already exists'
+        print('[INFO]: ' + consumer + ' already exists')
     else:
         myResult = createConsumer( consumer, salt, password )
         if myResult == '201':
-            print '[INFO]: ' + consumer + ' created, result: [' + myResult + ']'
+            print('[INFO]: ' + consumer + ' created, result: [' + myResult + ']')
         else:
-            print '[ERROR]: ' + bcolors.FAIL + consumer + bcolors.ENDC + ' error creating , result: [' + myResult + ']'
+            print('[ERROR]: ' + bcolors.FAIL + consumer + bcolors.ENDC + ' error creating , result: [' + myResult + ']')
diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/user.py.erb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/user.py.erb
deleted file mode 100644
index c2f267ba8b..0000000000
--- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/templates/default/user.py.erb
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/python
-import subprocess
-#from time import sleep
-import time
-from datetime import datetime
-
-BE_IP="<%= @be_ip %>"
-BE_PORT="<%= @be_port %>"
-PROTOCOL="<%= @protocol %>"
-
-class bcolors:
-    HEADER    = '\033[95m'
-    OKBLUE    = '\033[94m'
-    OKGREEN   = '\033[92m'
-    WARNING   = '\033[93m'
-    FAIL      = '\033[91m'
-    ENDC      = '\033[0m'
-    BOLD      = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-
-##############################
-#    Functions
-##############################
-def checkBackend():
-    command="curl -k -s -o /dev/null -I -w \"%{http_code}\" -i -H \"Accept: application/json; charset=UTF-8\" -H \"Content-Type: application/json\" -H \"USER_ID: jh0003\" "+PROTOCOL+"://" + BE_IP + ":" + BE_PORT + "/sdc2/rest/v1/user/jh0003"
-
-    proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE )
-    (out, err) = proc.communicate()
-    result = out.strip()
-    return result
-
-
-def checkUser(userName):
-    command="curl -k -s -o /dev/null -I -w \"%{http_code}\" -i -H \"Accept: application/json; charset=UTF-8\" -H \"Content-Type: application/json\" -H \"USER_ID: jh0003\" "+PROTOCOL+"://" + BE_IP + ":" + BE_PORT + "/sdc2/rest/v1/user/" + userName
-
-    proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE )
-    (out, err) = proc.communicate()
-    result = out.strip()
-    return result
-
-
-
-
-def createUser( firstName, lastName, userId , email_dom , role ):
-    print '[INFO] create first:[' + firstName + '], last:[' + lastName + '], Id:[' + userId + '], email:[' + userId + '@' + email_dom + '], role:[' + role +']'
-    command="curl -k -s -o /dev/null -w \"%{http_code}\" -X POST -i -H \"Accept: application/json; charset=UTF-8\" -H \"Content-Type: application/json\" -H \"USER_ID: jh0003\" "+PROTOCOL+"://" + BE_IP + ":" + BE_PORT + "/sdc2/rest/v1/user/ -d '{\"firstName\": \"" + firstName + "\", \"lastName\": \"" + lastName + "\",\"userId\": \"" + userId + "\",\"email\": \"" + userId + "@" + email_dom + "\",\"role\": \"" + role + "\"}'"
-
-    proc = subprocess.Popen( command , shell=True , stdout=subprocess.PIPE)
-    (out, err) = proc.communicate()
-    result = out.strip()
-    return result
-
-
-
-
-##############################
-#    Definitions
-##############################
-userId    = [ "demo" , "op0001" , "gv0001" , "jh0003" , "jm0007" , "cs0008" ]
-firstName = [ "demo" , "Oper" , "Giuseppe" , "Jimmy" , "Joni" , "Carlos" ]
-lastName  = [ "demo" , "P" , "Verdi" , "Hendrix" , "Mitchell" , "Santana" ]
-role      = [ "ADMIN" , "OPS" , "GOVERNOR" , "ADMIN" , "TESTER" , "DESIGNER" ]
-email_dom = "openecomp.org"
-beStat=0
-
-
-##############################
-#    Main
-##############################
-
-for i in range(1,10):
-    myResult = checkBackend()
-    if myResult == '200':
-        print '[INFO]: Backend is up and running'
-        beStat=1
-        break
-    else:
-        currentTime = datetime.now()
-        print '[ERROR]: ' + currentTime.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + ' Backend not responding, try #' + str(i) + bcolors.ENDC
-        time.sleep(10)
-
-if beStat == 0:
-    print '[ERROR]: ' + time.strftime('%Y/%m/%d %H:%M:%S') + bcolors.FAIL + 'Backend is DOWN :-(' + bcolors.ENDC
-    exit()
-
-for user in userId:
-    myResult = checkUser(user)
-    pos = userId.index(user)
-    if myResult == '200':
-        print '[INFO]: ' + user + ' already exists'
-    else:
-        myResult = createUser( firstName[pos], lastName[pos], userId[pos], email_dom, role[pos] )
-        if myResult == '201':
-            print '[INFO]: ' + userId[pos] + ' created, result: [' + myResult + ']'
-        else:
-            print '[ERROR]: ' + bcolors.FAIL + userId[pos] + bcolors.ENDC + ' error creating , result: [' + myResult + ']'
diff --git a/catalog-be/sdc-backend-init/startup.sh b/catalog-be/sdc-backend-init/startup.sh
index a379677bfb..83bfbf23de 100644
--- a/catalog-be/sdc-backend-init/startup.sh
+++ b/catalog-be/sdc-backend-init/startup.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-cd /home/sdc/chef-solo || exit $?
+cd /home/"${user}"/chef-solo || exit $?
 chef-solo -c solo.rb -E "${ENVNAME}"
 rc=$?
 if [ $rc != 0 ]; then exit $rc; fi