Refactor proxy API

refactor logger and auth method
fix identity proxy bug

Change-Id: Ie26c01c2fbf0869dcc77b8c182b775bce90b8d69
Issue-Id: MULTICLOUD-58
Signed-off-by: Bin Yang <bin.yang@windriver.com>

4 files changed, 89 insertions, 69 deletions

diff --git a/newton/newton/proxy/views/identityV3.py b/newton/newton/proxy/views/identityV3.py
index b06fb1e6..f11a6334 100644
--- a/newton/newton/proxy/views/identityV3.py
+++ b/newton/newton/proxy/views/identityV3.py
@@ -74,12 +74,19 @@ def update_catalog(vimid, catalog, multicould_namespace):
                     # replace the endpoint with MultiCloud's proxy
                     import re
                     endpoint_url = endpoint["url"]
-#                    m = re.search(r'^http[s]*://([0-9.]+:[0-9]+)[/]*([0-9a-zA-Z/._-]*)$', endpoint_url)
-                    m = re.search(r'^(http[s]?://[0-9.]+:[0-9]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
+                    real_prefix = None
+                    real_suffix = None
+#                    m = re.search(r'^(http[s]?://[0-9.]+:[0-9]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
+                    m = re.search(r'^(http[s]?://[0-9.]+[0-9:]*)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
                     if m:
                         real_prefix = m.group(1)
                         real_suffix = m.group(3)
-
+#                    else:
+#                        m = re.search(r'^(http[s]?://[0-9.]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
+#                        if m:
+#                            real_prefix = m.group(1)
+#                            real_suffix = m.group(2)
+                    if real_prefix:
                         # populate metadata_catalog
                         one_catalog['prefix'] = real_prefix
                         one_catalog['suffix'] = real_suffix if real_suffix else ''
@@ -96,6 +103,9 @@ def update_catalog(vimid, catalog, multicould_namespace):
 #                        endpoint["url"] = re.sub(r"^http([s]*)://([0-9.]+):([0-9]+)",
 #                                                 multicould_namespace + "/%s/" % vimid + item["type"],
 #                                                 endpoint["url"])
+                    else:
+                        #something wrong
+                        pass
 
 
                     endpoint["url"] = endpoint_url
@@ -114,9 +124,10 @@ class Tokens(APIView):
 
     def __init__(self):
         self.proxy_prefix = MULTICLOUD_PREFIX
+        self._logger = logger
 
     def post(self, request, vimid=""):
-        logger.debug("identityV3--post::> %s" % request.data)
+        self._logger.debug("identityV3--post::> %s" % request.data)
         sess = None
         resp = None
         resp_body = None
@@ -135,7 +146,7 @@ class Tokens(APIView):
 
             #update the catalog
             tmp_auth_data['token']['catalog'], tmp_metadata_catalog = update_catalog(vimid, tmp_auth_data['token']['catalog'], self.proxy_prefix)
-            VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state, json.dumps(tmp_metadata_catalog))
+            tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state, json.dumps(tmp_metadata_catalog))
 
             resp = Response(headers={'X-Subject-Token': tmp_auth_token}, data=tmp_auth_data, status=status.HTTP_201_CREATED)
             return resp
@@ -143,10 +154,10 @@ class Tokens(APIView):
 
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
 
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
@@ -159,10 +170,11 @@ class Catalog(APIView):
 
     def __init__(self):
         self.proxy_prefix = MULTICLOUD_PREFIX
+        self._logger = logger
 
     def get(self, request, vimid=""):
-        logger.debug("Catalog--get::data> %s" % request.data)
-#        logger.debug("Catalog--get::META> %s" % request.META)
+        self._logger.debug("Catalog--get::data> %s" % request.data)
+#        self._logger.debug("Catalog--get::META> %s" % request.META)
         try:
             # prepare request resource to vim instance
             #get token:
@@ -191,9 +203,9 @@ class Catalog(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/newton/newton/proxy/views/services.py b/newton/newton/proxy/views/services.py
index 1ca8b535..1a478f35 100644
--- a/newton/newton/proxy/views/services.py
+++ b/newton/newton/proxy/views/services.py
@@ -35,10 +35,13 @@ DEBUG=True
 
 class Services(APIView):
 
+    def __init__(self):
+        self._logger = logger
+
     def head(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--head::META> %s" % request.META)
-        logger.debug("Services--head::data> %s" % request.data)
-        logger.debug("Services--head::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("Services--head::META> %s" % request.META)
+        self._logger.debug("Services--head::data> %s" % request.data)
+        self._logger.debug("Services--head::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
 
         try:
@@ -75,17 +78,17 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def get(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--get::META> %s" % request.META)
-        logger.debug("Services--get::data> %s" % request.data)
-        logger.debug("Services--get::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("Services--get::META> %s" % request.META)
+        self._logger.debug("Services--get::data> %s" % request.data)
+        self._logger.debug("Services--get::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
         try:
             # prepare request resource to vim instance
@@ -104,7 +107,7 @@ class Services(APIView):
             proxy_prefix = None
             suffix = None
             if servicetype and metadata_catalog:
-#                logger.error("metadata_catalog:%s" % metadata_catalog)
+#                self._logger.error("metadata_catalog:%s" % metadata_catalog)
                 metadata_catalog = json.loads(metadata_catalog)
                 service_metadata = metadata_catalog.get(servicetype, None)
                 if service_metadata:
@@ -152,17 +155,17 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def post(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--post::META> %s" % request.META)
-        logger.debug("Services--post::data> %s" % request.data)
-        logger.debug("Services--post::vimid, servicetype,  requri> %s,%s,%s"
+        self._logger.debug("Services--post::META> %s" % request.META)
+        self._logger.debug("Services--post::data> %s" % request.data)
+        self._logger.debug("Services--post::vimid, servicetype,  requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
         try:
             # prepare request resource to vim instance
@@ -182,7 +185,7 @@ class Services(APIView):
             proxy_prefix = None
             suffix = None
             if servicetype and metadata_catalog:
-#                logger.error("metadata_catalog:%s" % metadata_catalog)
+#                self._logger.error("metadata_catalog:%s" % metadata_catalog)
                 metadata_catalog = json.loads(metadata_catalog)
                 service_metadata = metadata_catalog.get(servicetype, None)
                 if service_metadata:
@@ -230,17 +233,17 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def put(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--put::META> %s" % request.META)
-        logger.debug("Services--put::data> %s" % request.data)
-        logger.debug("Services--put::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("Services--put::META> %s" % request.META)
+        self._logger.debug("Services--put::data> %s" % request.data)
+        self._logger.debug("Services--put::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
         try:
             # prepare request resource to vim instance
@@ -260,7 +263,7 @@ class Services(APIView):
             proxy_prefix = None
             suffix = None
             if servicetype and metadata_catalog:
-#                logger.error("metadata_catalog:%s" % metadata_catalog)
+#                self._logger.error("metadata_catalog:%s" % metadata_catalog)
                 metadata_catalog = json.loads(metadata_catalog)
                 service_metadata = metadata_catalog.get(servicetype, None)
                 if service_metadata:
@@ -308,18 +311,18 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
 
     def patch(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--patch::META> %s" % request.META)
-        logger.debug("Services--patch::data> %s" % request.data)
-        logger.debug("Services--patch::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("Services--patch::META> %s" % request.META)
+        self._logger.debug("Services--patch::data> %s" % request.data)
+        self._logger.debug("Services--patch::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
         try:
             # prepare request resource to vim instance
@@ -339,7 +342,7 @@ class Services(APIView):
             proxy_prefix = None
             suffix = None
             if servicetype and metadata_catalog:
-#                logger.error("metadata_catalog:%s" % metadata_catalog)
+#                self._logger.error("metadata_catalog:%s" % metadata_catalog)
                 metadata_catalog = json.loads(metadata_catalog)
                 service_metadata = metadata_catalog.get(servicetype, None)
                 if service_metadata:
@@ -387,17 +390,17 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def delete(self, request, vimid="", servicetype="", requri=""):
-        logger.debug("Services--delete::META> %s" % request.META)
-        logger.debug("Services--delete::data> %s" % request.data)
-        logger.debug("Services--delete::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("Services--delete::META> %s" % request.META)
+        self._logger.debug("Services--delete::data> %s" % request.data)
+        self._logger.debug("Services--delete::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
         try:
             # prepare request resource to vim instance
@@ -417,7 +420,7 @@ class Services(APIView):
             proxy_prefix = None
             suffix = None
             if servicetype and metadata_catalog:
-#                logger.error("metadata_catalog:%s" % metadata_catalog)
+#                self._logger.error("metadata_catalog:%s" % metadata_catalog)
                 metadata_catalog = json.loads(metadata_catalog)
                 service_metadata = metadata_catalog.get(servicetype, None)
                 if service_metadata:
@@ -458,10 +461,10 @@ class Services(APIView):
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
-            logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
             return Response(data=e.response.json(), status=e.http_status)
         except Exception as e:
-            logger.error(traceback.format_exc())
+            self._logger.error(traceback.format_exc())
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
@@ -470,10 +473,14 @@ class GetTenants(Services):
     '''
     Backward compatible API for /v2.0/tenants
     '''
+
+    def __init__(self):
+        self._logger = logger
+
     def get(self, request, vimid="", servicetype="identity", requri='projects'):
-        logger.debug("GetTenants--get::META> %s" % request.META)
-        logger.debug("GetTenants--get::data> %s" % request.data)
-        logger.debug("GetTenants--get::vimid, servicetype, requri> %s,%s,%s"
+        self._logger.debug("GetTenants--get::META> %s" % request.META)
+        self._logger.debug("GetTenants--get::data> %s" % request.data)
+        self._logger.debug("GetTenants--get::vimid, servicetype, requri> %s,%s,%s"
                      % (vimid, servicetype, requri))
 
         tmp_auth_token = request.META.get('HTTP_X_AUTH_TOKEN', None)
diff --git a/newton/newton/requests/views/util.py b/newton/newton/requests/views/util.py
index b94a9ed3..c5046387 100644
--- a/newton/newton/requests/views/util.py
+++ b/newton/newton/requests/views/util.py
@@ -71,6 +71,11 @@ class VimDriverUtils(object):
                                             username=vim["userName"],
                                             password=vim["password"],
                                             project_id=tenantid)
+            elif '/identity' in vim["url"]:
+                auth = keystone_v3.Password(auth_url=vim["url"]+"/v3",
+                                            username=vim["userName"],
+                                            password=vim["password"],
+                                            project_id=tenantid)
         elif tenant_name:
             if '/v2' in vim["url"]:
                 auth = keystone_v2.Password(auth_url=vim["url"],
@@ -84,6 +89,13 @@ class VimDriverUtils(object):
                                             project_name=tenant_name,
                                             user_domain_name=vim["domain"],
                                             project_domain_name=vim["domain"])
+            elif '/identity' in vim["url"]:
+                auth = keystone_v3.Password(auth_url=vim["url"]+"/v3",
+                                            username=vim["userName"],
+                                            password=vim["password"],
+                                            project_name=tenant_name,
+                                            user_domain_name=vim["domain"],
+                                            project_domain_name=vim["domain"])
 
         else:
             #something wrong
@@ -124,40 +136,30 @@ class VimDriverUtils(object):
 
 
     @staticmethod
-    def update_token_cache(vim, session, old_token, auth_state, metadata=None):
+    def update_token_cache(vim, session, token, auth_state, metadata=None):
         '''
         cache the auth_state as well as metadata_catalog
         :param vim:
         :param session:
-        :param old_token:
+        :param token:
         :param auth_state:
         :param matadata:
         :return:
         '''
 
-        metadata_key = "meta_%s" % old_token
+        if metadata == None: #do not update token any more
+            return token
 
-        if not metadata:
-            metadata = cache.get(metadata_key)
-        if not metadata:
-            #something wrong since metadata is neither inputted, nor cached previously. but ignore temporarily
-            pass
-
-        tmp_auth_token = session.get_token()
-        #check if need to update token:auth_state mapping
-        if tmp_auth_token != old_token:
-             cache.delete(old_token)
-             cache.delete(metadata_key)
-             metadata_key = "meta_%s" % tmp_auth_token
+        metadata_key = "meta_%s" % token
 
-        elif not cache.get(old_token):
+        if not cache.get(token):
             # store the auth_state, memcached
             # set expiring in 1 hour
-            cache.set(tmp_auth_token, auth_state, 3600)
+            cache.set(token, auth_state, 3600)
             cache.set(metadata_key, metadata, 3600)
 
-        #return new token
-        return tmp_auth_token
+        return token
+
 
     @staticmethod
     def replace_a_key(dict_obj, keypair, reverse=False):
diff --git a/newton/newton/settings.py b/newton/newton/settings.py
index 682e8eb4..b62a584f 100644
--- a/newton/newton/settings.py
+++ b/newton/newton/settings.py
@@ -35,7 +35,6 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
-    'newton.pub.database',
 ]
 
 MIDDLEWARE_CLASSES = [