fix vcpeutils

removed relative imports and using the preferred method of importing.
use the open instead of file method to get files
use the new pyaml method that specifies the loader
use builtins.basestring for unicode and nonunicode comparison
use the correct method to call the robot framework logger warn
adding in unit tests for things to make sure they work

Change-Id: I15da74ff66988ef2610ada3ae21a1c6104c26c35
Issue-ID: INT-1061
Signed-off-by: DR695H <dr695h@att.com>

10 files changed, 90 insertions, 78 deletions

diff --git a/robotframework-onap/setup.py b/robotframework-onap/setup.py
index 969fe9d..2307375 100644
--- a/robotframework-onap/setup.py
+++ b/robotframework-onap/setup.py
@@ -33,12 +33,18 @@ setup(
         'robotframework',
         'deepdiff',
         'Jinja2',
-        'urllib3',
+        # 'urllib3', # requests gets upset if this is specified since they only work with a very
+        # narrow range of urlib3. if for some reason we remove requests, readd this back in
         'six',
-        'requests'
+        'requests',
+        'future'
     ],  # what we need
     packages=['eteutils', 'loadtest', 'vcpeutils'],       # The name of your scripts package
-    package_dir={'eteutils': 'eteutils', 'loadtest': 'loadtest', 'vcpeutils':'vcpeutils'}, # The location of your scipts package
+    package_dir={
+        'eteutils': 'eteutils',
+        'loadtest': 'loadtest',
+        'vcpeutils': 'vcpeutils'
+    },  # The location of your scipts package
     classifiers=[
         'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
@@ -48,5 +54,6 @@ setup(
         'Framework :: Robot Framework',
         'Framework :: Robot Framework :: Library',
         'License :: OSI Approved :: Apache Software License'
-    ]
+    ],
+    test_suite="tests.runner"
 )
diff --git a/robotframework-onap/tests/__init__.py b/robotframework-onap/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/robotframework-onap/tests/__init__.py
diff --git a/robotframework-onap/tests/runner.py b/robotframework-onap/tests/runner.py
new file mode 100644
index 0000000..b58215a
--- /dev/null
+++ b/robotframework-onap/tests/runner.py
@@ -0,0 +1,16 @@
+import unittest
+
+# import your test modules
+from tests.vcpeutils.SoUtils_test import *
+
+
+# initialize the test suite
+loader = unittest.TestLoader()
+suite = unittest.TestSuite()
+
+# add tests to the test suite
+suite.addTests(loader.loadTestsFromTestCase(SoUtilsTest))
+
+# initialize a runner, pass it your suite and run it
+runner = unittest.TextTestRunner(verbosity=3)
+result = runner.run(suite)
diff --git a/robotframework-onap/tests/vcpeutils/SoUtils_test.py b/robotframework-onap/tests/vcpeutils/SoUtils_test.py
new file mode 100644
index 0000000..22afed6
--- /dev/null
+++ b/robotframework-onap/tests/vcpeutils/SoUtils_test.py
@@ -0,0 +1,14 @@
+from unittest import TestCase
+
+
+from vcpeutils.SoUtils import *
+
+
+class SoUtilsTest(TestCase):
+
+    def test(self):
+        input_dict = dict()
+        SoUtils.add_related_instance(input_dict, "test", "test2")
+        results = dict()
+        results['relatedInstanceList'] = [{"relatedInstance": {"instanceId": "test", "modelInfo": "test2"}}]
+        self.assertDictContainsSubset(results, input_dict)
diff --git a/robotframework-onap/tests/vcpeutils/__init__.py b/robotframework-onap/tests/vcpeutils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/robotframework-onap/tests/vcpeutils/__init__.py
diff --git a/robotframework-onap/tox.ini b/robotframework-onap/tox.ini
index d9a097f..bd1bff5 100644
--- a/robotframework-onap/tox.ini
+++ b/robotframework-onap/tox.ini
@@ -4,9 +4,8 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, py36
+envlist = py27, py3
 
 [testenv]
 commands =  {envpython} setup.py test
-deps =
-
+deps =
\ No newline at end of file
diff --git a/robotframework-onap/vcpeutils/SoUtils.py b/robotframework-onap/vcpeutils/SoUtils.py
index c52006d..02b34e5 100755
--- a/robotframework-onap/vcpeutils/SoUtils.py
+++ b/robotframework-onap/vcpeutils/SoUtils.py
@@ -2,26 +2,22 @@
 
 import time
 
-from .csar_parser import *
-from .preload import *
-from .vcpecommon import *
+from vcpeutils.preload import *
+from vcpeutils.vcpecommon import *
+
 from robot.api import logger
 
 
 class SoUtils:
 
     def __init__(self):
-        """
-        :param api_version: must be 'v4' or 'v5'
-        """
-        self.tmp_solution_for_so_bug = False
-        #self.logger = logging.getLogger(__name__)
-        self.logger = logger 
+        self.region_name = None  # set later
+        self.tenant_id = None  # set later
+        self.logger = logger
         self.vcpecommon = VcpeCommon()
         self.api_version = 'v4'
         self.service_req_api_url = self.vcpecommon.so_req_api_url[self.api_version]
 
-
     def submit_create_req(self, req_json, req_type, service_instance_id=None, vnf_instance_id=None):
         """
         POST	{serverRoot}/serviceInstances/v4
@@ -60,12 +56,11 @@ class SoUtils:
 
         return req_id, instance_id
 
-    def check_progress(self, req_id, eta=0, interval=5):
+    def check_progress(self, req_id, interval=5):
         if not req_id:
             self.logger.error('Error when checking SO request progress, invalid request ID: ' + req_id)
             return False
         duration = 0.0
-        #bar = progressbar.ProgressBar(redirect_stdout=True)
         url = self.vcpecommon.so_check_progress_api_url + '/' + req_id
 
         while True:
@@ -74,15 +69,10 @@ class SoUtils:
             response = r.json()
 
             duration += interval
-            if eta > 0:
-                percentage = min(95, 100 * duration / eta)
-            else:
-                percentage = int(response['request']['requestStatus']['percentProgress'])
 
             if response['request']['requestStatus']['requestState'] == 'IN_PROGRESS':
                 self.logger.debug('------------------Request Status-------------------------------')
                 self.logger.debug(json.dumps(response, indent=4, sort_keys=True))
-                #bar.update(percentage)
             else:
                 self.logger.debug('---------------------------------------------------------------')
                 self.logger.debug('----------------- Creation Request Results --------------------')
@@ -92,11 +82,10 @@ class SoUtils:
                 if not flag:
                     self.logger.error('Request failed.')
                     self.logger.error(json.dumps(response, indent=4, sort_keys=True))
-                #bar.update(100)
-                #bar.finish()
                 return flag
 
-    def add_req_info(self, req_details, instance_name, product_family_id=None):
+    @staticmethod
+    def add_req_info(req_details, instance_name, product_family_id=None):
         req_details['requestInfo'] = {
                     'instanceName': instance_name,
                     'source': 'VID',
@@ -106,19 +95,17 @@ class SoUtils:
         if product_family_id:
             req_details['requestInfo']['productFamilyId'] = product_family_id
 
-    def add_related_instance(self, req_details, instance_id, instance_model):
+    @staticmethod
+    def add_related_instance(req_details, instance_id, instance_model):
         instance = {"instanceId": instance_id, "modelInfo": instance_model}
         if 'relatedInstanceList' not in req_details:
             req_details['relatedInstanceList'] = [{"relatedInstance": instance}]
         else:
             req_details['relatedInstanceList'].append({"relatedInstance": instance})
 
-    def generate_vnf_or_network_request(self, req_type, instance_name, vnf_or_network_model, service_instance_id,
-                                        service_model):
+    def generate_vnf_or_network_request(self, instance_name, vnf_or_network_model, service_instance_id, service_model):
         req_details = {
             'modelInfo':  vnf_or_network_model,
-            #'cloudConfiguration': {"lcpCloudRegionId": self.vcpecommon.os_region_name,
-            #                       "tenantId": self.vcpecommon.os_tenant_id},
             'cloudConfiguration': {"lcpCloudRegionId": self.region_name,
                                    "tenantId": self.tenant_id},
             'requestParameters':  {"userParams": []},
@@ -129,11 +116,9 @@ class SoUtils:
         return {'requestDetails': req_details}
 
     def generate_vfmodule_request(self, instance_name, vfmodule_model, service_instance_id,
-                                        service_model, vnf_instance_id, vnf_model):
+                                  service_model, vnf_instance_id, vnf_model):
         req_details = {
             'modelInfo':  vfmodule_model,
-            #'cloudConfiguration': {"lcpCloudRegionId": self.vcpecommon.os_region_name,
-            #                       "tenantId": self.vcpecommon.os_tenant_id},
             'cloudConfiguration': {"lcpCloudRegionId": self.region_name,
                                    "tenantId": self.tenant_id},
             'requestParameters': {"usePreload": 'true'}
@@ -223,35 +208,33 @@ class SoUtils:
         self.logger.info(json.dumps(req, indent=2, sort_keys=True))
         self.logger.info('Creating custom service {0}.'.format(instance_name))
         req_id, svc_instance_id = self.submit_create_req(req, 'service')
-        if not self.check_progress(req_id, 140):
+        if not self.check_progress(req_id):
             return False
         return True
 
     def wait_for_aai(self, node_type, uuid):
         self.logger.info('Waiting for AAI traversal to complete...')
-        #bar = progressbar.ProgressBar()
         for i in range(30):
             time.sleep(1)
-            #bar.update(i*100.0/30)
             if self.vcpecommon.is_node_in_aai(node_type, uuid):
-                #bar.update(100)
-                #bar.finish()
                 return
 
         self.logger.error("AAI traversal didn't finish in 30 seconds. Something is wrong. Type {0}, UUID {1}".format(
             node_type, uuid))
         sys.exit()
 
-    def create_entire_service(self, csar_file, vnf_template_file, preload_dict, name_suffix, region_name, tenant_id, heatbridge=False):
+    def create_entire_service(self, csar_file, vnf_template_file, preload_dict, name_suffix, region_name, tenant_id):
         """
         :param csar_file:
         :param vnf_template_file:
         :param preload_dict:
         :param name_suffix:
+        :param region_name:
+        :param tenant_id
         :return:  service instance UUID
         """
-        self.region_name=region_name
-        self.tenant_id=tenant_id
+        self.region_name = region_name
+        self.tenant_id = tenant_id
         self.logger.info('\n----------------------------------------------------------------------------------')
         self.logger.info('Start to create entire service defined in csar: {0}'.format(csar_file))
         parser = CsarParser()
@@ -266,13 +249,13 @@ class SoUtils:
         instance_name = '_'.join([self.vcpecommon.instance_name_prefix['service'],
                                   parser.svc_model['modelName'], global_timestamp, name_suffix])
         instance_name = instance_name.lower()
-        instance_name = instance_name.replace(' ','')
-        instance_name = instance_name.replace(':','')
+        instance_name = instance_name.replace(' ', '')
+        instance_name = instance_name.replace(':', '')
         self.logger.info('Creating service instance: {0}.'.format(instance_name))
         req = self.generate_service_request(instance_name, parser.svc_model)
         self.logger.debug(json.dumps(req, indent=2, sort_keys=True))
         req_id, svc_instance_id = self.submit_create_req(req, 'service')
-        if not self.check_progress(req_id, eta=2, interval=5):
+        if not self.check_progress(req_id, interval=5):
             return None
 
         # wait for AAI to complete traversal
@@ -284,11 +267,10 @@ class SoUtils:
             network_name = '_'.join([self.vcpecommon.instance_name_prefix['network'], base_name, name_suffix])
             network_name = network_name.lower()
             self.logger.info('Creating network: ' + network_name)
-            req = self.generate_vnf_or_network_request('network', network_name, model, svc_instance_id,
-                                                       parser.svc_model)
+            req = self.generate_vnf_or_network_request(network_name, model, svc_instance_id, parser.svc_model)
             self.logger.debug(json.dumps(req, indent=2, sort_keys=True))
             req_id, net_instance_id = self.submit_create_req(req, 'network', svc_instance_id)
-            if not self.check_progress(req_id, eta=20):
+            if not self.check_progress(req_id):
                 return None
 
             self.logger.info('Changing subnet name to ' + self.vcpecommon.network_name_to_subnet_name(network_name))
@@ -304,7 +286,6 @@ class SoUtils:
                 self.logger.error('Failed to change subnet name for ' + network_name)
                 return None
 
-
         vnf_model = None
         vnf_instance_id = None
         # create VNF
@@ -313,14 +294,13 @@ class SoUtils:
             vnf_instance_name = '_'.join([self.vcpecommon.instance_name_prefix['vnf'],
                                           vnf_model['modelCustomizationName'].split(' ')[0],  name_suffix])
             vnf_instance_name = vnf_instance_name.lower()
-            vnf_instance_name = vnf_instance_name.replace(' ','')
-            vnf_instance_name = vnf_instance_name.replace(':','')
+            vnf_instance_name = vnf_instance_name.replace(' ', '')
+            vnf_instance_name = vnf_instance_name.replace(':', '')
             self.logger.info('Creating VNF: ' + vnf_instance_name)
-            req = self.generate_vnf_or_network_request('vnf', vnf_instance_name, vnf_model, svc_instance_id,
-                                                       parser.svc_model)
+            req = self.generate_vnf_or_network_request(vnf_instance_name, vnf_model, svc_instance_id, parser.svc_model)
             self.logger.debug(json.dumps(req, indent=2, sort_keys=True))
             req_id, vnf_instance_id = self.submit_create_req(req, 'vnf', svc_instance_id)
-            if not self.check_progress(req_id, eta=2, interval=5):
+            if not self.check_progress(req_id, interval=5):
                 self.logger.error('Failed to create VNF {0}.'.format(vnf_instance_name))
                 return False
 
@@ -346,21 +326,15 @@ class SoUtils:
             vfmodule_instance_name = '_'.join([self.vcpecommon.instance_name_prefix['vfmodule'],
                                                model['modelCustomizationName'].split('..')[0], name_suffix])
             vfmodule_instance_name = vfmodule_instance_name.lower()
-            vfmodule_instance_name = vfmodule_instance_name.replace(' ','')
-            vfmoduel_instance_name = vfmodule_instance_name.replace(':','')
+            vfmodule_instance_name = vfmodule_instance_name.replace(' ', '')
+            vfmodule_instance_name = vfmodule_instance_name.replace(':', '')
             self.logger.info('Creating VF Module: ' + vfmodule_instance_name)
             req = self.generate_vfmodule_request(vfmodule_instance_name, model, svc_instance_id, parser.svc_model,
                                                  vnf_instance_id, vnf_model)
             self.logger.debug(json.dumps(req, indent=2, sort_keys=True))
             req_id, vfmodule_instance_id = self.submit_create_req(req, 'vfmodule', svc_instance_id, vnf_instance_id)
-            if not self.check_progress(req_id, eta=70, interval=50):
+            if not self.check_progress(req_id, interval=50):
                 self.logger.error('Failed to create VF Module {0}.'.format(vfmodule_instance_name))
                 return None
 
-        # run heatbridge
-        # removed until we fold in heatbridge
-        #if heatbridge:
-            #self.vcpecommon.heatbridge(vfmodule_instance_name, svc_instance_id)
-            #self.vcpecommon.save_vgmux_vnf_name(vnf_instance_name)
-
-        return svc_instance_id
+        return svc_instance_id
\ No newline at end of file
diff --git a/robotframework-onap/vcpeutils/csar_parser.py b/robotframework-onap/vcpeutils/csar_parser.py
index 7440310..2adcf63 100755
--- a/robotframework-onap/vcpeutils/csar_parser.py
+++ b/robotframework-onap/vcpeutils/csar_parser.py
@@ -12,8 +12,8 @@ class CsarParser:
         self.logger = logging.getLogger(__name__)
         self.svc_model = {}
         self.net_models = []  # there could be multiple networks
-        self.vnf_models = [] # this version only support a single VNF in the service template
-        self.vfmodule_models = [] # this version only support a single VF module in the service template
+        self.vnf_models = []  # this version only support a single VNF in the service template
+        self.vfmodule_models = []  # this version only support a single VF module in the service template
 
     def get_service_yaml_from_csar(self, csar_file):
         """
@@ -198,7 +198,7 @@ class CsarParser:
         self.vnf_models = []    # this version only support a single VNF in the service template
         self.vfmodule_models = []   # this version only support a single VF module in the service template
 
-        svc_template = yaml.load(file(filename, 'r'))
+        svc_template = yaml.load(open(filename, 'r'), Loader=yaml.Loader)
         self.get_service_model_info(svc_template)
         self.get_vnf_and_network_model_info(svc_template)
         self.get_vfmodule_model_info(svc_template)
@@ -212,19 +212,19 @@ class CsarParser:
 
     def print_models(self):
         print('---------Service Model----------')
-        print((json.dumps(self.svc_model, indent=2, sort_keys=True)))
+        print(json.dumps(self.svc_model, indent=2, sort_keys=True))
 
         print('---------Network Model(s)----------')
         for model in self.net_models:
-            print((json.dumps(model, indent=2, sort_keys=True)))
+            print(json.dumps(model, indent=2, sort_keys=True))
 
         print('---------VNF Model(s)----------')
         for model in self.vnf_models:
-            print((json.dumps(model, indent=2, sort_keys=True)))
+            print(json.dumps(model, indent=2, sort_keys=True))
 
         print('---------VF Module Model(s)----------')
         for model in self.vfmodule_models:
-            print((json.dumps(model, indent=2, sort_keys=True)))
+            print(json.dumps(model, indent=2, sort_keys=True))
 
     def test(self):
         self.parse_csar('csar/service-Vcpesvcinfra111601-csar.csar')
diff --git a/robotframework-onap/vcpeutils/preload.py b/robotframework-onap/vcpeutils/preload.py
index 6ccc689..642b5e7 100755
--- a/robotframework-onap/vcpeutils/preload.py
+++ b/robotframework-onap/vcpeutils/preload.py
@@ -1,15 +1,15 @@
 #! /usr/bin/python
 
 from datetime import datetime
-from .vcpecommon import *
-from .csar_parser import *
+from vcpeutils.vcpecommon import *
+from vcpeutils.csar_parser import *
 from robot.api import logger
+from past import builtins
 import base64
 
 
 class Preload:
     def __init__(self, vcpecommon):
-        #self.logger = logging.getLogger(__name__)
         self.logger = logger
         self.vcpecommon = vcpecommon
 
@@ -31,11 +31,11 @@ class Preload:
                         stk.append(v)
                     elif type(v) is list:
                         stk.extend(v)
-                    elif type(v) is str or type(v) is str:
+                    elif type(v) is builtins.basestring:
                         if self.vcpecommon.template_variable_symbol in v:
                             data[k] = self.replace(v, replace_dict)
                     else:
-                        self.logger.warning('Unexpected line in template: %s. Look for value %s', template_file, v)
+                        self.logger.warn('Unexpected line in template: {}. Look for value {}'.format(template_file, v))
         return json_data
 
     def reset_sniro(self):
@@ -83,6 +83,7 @@ class Preload:
         :param network_role: cpe_signal, cpe_public, brg_bng, bng_mux, mux_gw
         :param subnet_start_ip:
         :param subnet_gateway:
+        :param common_dict:
         :param name_suffix: e.g. '201711201311'
         :return:
         """
@@ -184,7 +185,7 @@ class Preload:
 
         print('---------------------------------------------------------------')
         print('Network related replacement dictionary:')
-        print((json.dumps(network_dict, indent=4, sort_keys=True)))
+        print(json.dumps(network_dict, indent=4, sort_keys=True))
         print('---------------------------------------------------------------')
 
         keys = ['infra', 'bng', 'gmux', 'brg']
diff --git a/robotframework-onap/vcpeutils/vcpecommon.py b/robotframework-onap/vcpeutils/vcpecommon.py
index 0d45ee3..c629b40 100755
--- a/robotframework-onap/vcpeutils/vcpecommon.py
+++ b/robotframework-onap/vcpeutils/vcpecommon.py
@@ -8,6 +8,7 @@ import sys
 import ipaddress
 import requests
 
+
 class VcpeCommon:
     #############################################################################################
     #     Start: configurations that you must change for a new ONAP installation