Refactor newton and ocato proxy

refactor code and fix bugs to update proxy prefix

Change-Id: I4a02b0570759f087581e875cc683d4475bd2121b
Issue-Id: MULTICLOUD-89
Signed-off-by: Bin Yang <bin.yang@windriver.com>

10 files changed, 155 insertions, 215 deletions

diff --git a/newton/docker/Dockerfile b/newton/docker/Dockerfile
index 51379089..fb624cd4 100644
--- a/newton/docker/Dockerfile
+++ b/newton/docker/Dockerfile
@@ -23,6 +23,7 @@ RUN apt-get update && \
     cd /opt/ && \
     wget -O multicloud-openstack-newton.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=snapshots&g=org.onap.multicloud.openstack&a=multicloud-openstack-newton&e=zip&v=LATEST" && \
     unzip -q -o -B multicloud-openstack-newton.zip && \
+    chmod +x /opt/newton/*.sh && \
     rm -f multicloud-openstack-newton.zip && \
     pip install -r /opt/newton/requirements.txt
 
diff --git a/newton/docker/build-image.sh b/newton/docker/build-image.sh
index fd8fb8cd..fd8fb8cd 100755..100644
--- a/newton/docker/build-image.sh
+++ b/newton/docker/build-image.sh
diff --git a/newton/newton/extensions/views/epacaps.py b/newton/newton/extensions/views/epacaps.py
index ef8aabed..54bd0ebe 100644
--- a/newton/newton/extensions/views/epacaps.py
+++ b/newton/newton/extensions/views/epacaps.py
@@ -34,6 +34,10 @@ DEBUG=True
 
 class EpaCaps(APIView):
 
+    def __init__(self):
+        self.proxy_prefix = config.MULTICLOUD_PREFIX
+        self._logger = logger
+
     def get(self, request, vimid=""):
         logger.debug("EpaCaps--get::data> %s" % request.data)
         logger.debug("EpaCaps--get::vimid> %s"
diff --git a/newton/newton/extensions/views/extensions.py b/newton/newton/extensions/views/extensions.py
index e1901d01..a196edcc 100644
--- a/newton/newton/extensions/views/extensions.py
+++ b/newton/newton/extensions/views/extensions.py
@@ -36,6 +36,10 @@ DEBUG=True
 
 class Extensions(APIView):
 
+    def __init__(self):
+        self.proxy_prefix = config.MULTICLOUD_PREFIX
+        self._logger = logger
+
     def get(self, request, vimid=""):
         logger.debug("Extensions--get::data> %s" % request.data)
         logger.debug("Extensions--get::vimid> %s"
@@ -48,7 +52,7 @@ class Extensions(APIView):
                         "alias": "epa-caps",
                         "description": "Multiple network support",
                         "name": "EPACapsQuery",
-                        "url": config.MULTICLOUD_PREFIX+"/%s/extensions/epa-caps" \
+                        "url": self.proxy_prefix+"/%s/extensions/epa-caps" \
                                        % (vimid),
                         "spec": ""
                     }
@@ -61,7 +65,7 @@ class Extensions(APIView):
                 "extensions": registered_extensions
             }
             return Response(data=content, status=status.HTTP_200_OK)
-            #return resp
+
         except VimDriverNewtonException as e:
             return Response(data={'error': e.content}, status=e.status_code)
         except HttpError as e:
diff --git a/newton/newton/proxy/tests/test_identity_proxy.py b/newton/newton/proxy/tests/test_identity_proxy.py
index a61113ee..6fb85b2c 100644
--- a/newton/newton/proxy/tests/test_identity_proxy.py
+++ b/newton/newton/proxy/tests/test_identity_proxy.py
@@ -23,7 +23,7 @@ from keystoneauth1 import session
 from keystoneauth1.exceptions import HttpError
 
 from newton.requests.views.util import VimDriverUtils
-from newton.proxy.views.identityV3 import Tokens, Catalog
+from newton.proxy.views.identityV3 import Tokens
 
 mock_viminfo = {
     "createTime": "2017-04-01 02:22:27",
@@ -494,14 +494,6 @@ mock_auth_state = {
 }
 
 
-
-class mock_catalog_response_specs(object):
-   status_code = 200
-
-   def json(self):
-      return mock_catalog_response
-
-
 class TestIdentityService(unittest.TestCase):
     def setUp(self):
         self.client = Client()
@@ -539,43 +531,3 @@ class TestIdentityService(unittest.TestCase):
 
         self.assertTrue(response['X-Subject-Token'] == mock_token_id)
         self.assertTrue(context['token']['catalog'] != None)
-
-
-    @mock.patch.object(VimDriverUtils, 'get_vim_info')
-    @mock.patch.object(VimDriverUtils, 'get_session')
-    @mock.patch.object(VimDriverUtils, 'get_auth_state')
-    @mock.patch.object(VimDriverUtils, 'update_token_cache')
-    @mock.patch.object(VimDriverUtils, 'get_token_cache')
-    def test_catalog(self, mock_get_token_cache, mock_update_token_cache, mock_get_auth_state, mock_get_session, mock_get_vim_info):
-        '''
-                test API: get token
-        :param mock_update_token_cache:
-        :param mock_get_auth_state:
-        :param mock_get_session:
-        :param mock_get_vim_info:
-        :return:
-        '''
-
-        #mock VimDriverUtils APIs
-        mock_session_specs = ["get"]
-
-        mock_session = mock.Mock(name='mock_session', spec=mock_session_specs)
-        mock_catalog_response_obj = mock.Mock(spec=mock_catalog_response_specs)
-        mock_catalog_response_obj.status_code=200
-        mock_catalog_response_obj.json.return_value=mock_catalog_response
-        mock_session.get.return_value = mock_catalog_response_obj
-
-        mock_get_vim_info.return_value = mock_viminfo
-        mock_get_session.return_value = mock_session
-        mock_get_auth_state.return_value = json.dumps(mock_auth_state)
-        mock_update_token_cache.return_value = mock_token_id
-        mock_get_token_cache.return_value = (json.dumps(mock_auth_state),{})
-
-        #simulate client to make the request
-        response = self.client.get("/api/multicloud-newton/v0/windriver-hudson-dc_RegionOne/identity/v3/auth/catalog",{},
-                                   HTTP_X_AUTH_TOKEN=mock_token_id)
-        self.failUnlessEqual(status.HTTP_200_OK, response.status_code)
-        context = response.json()
-
-        self.assertTrue(response['X-Subject-Token'] == mock_token_id)
-        self.assertTrue(context['catalog'] != None)
diff --git a/newton/newton/proxy/urls.py b/newton/newton/proxy/urls.py
index 99a4e851..ae63c052 100644
--- a/newton/newton/proxy/urls.py
+++ b/newton/newton/proxy/urls.py
@@ -23,8 +23,6 @@ urlpatterns = [
     #        identityV2.Tokens.as_view()),
     url(r'^identity/v3/auth/tokens$',
         identityV3.Tokens.as_view()),
-    url(r'^identity/v3/auth/catalog$',
-        identityV3.Catalog.as_view()),
     url(r'^identity/(?:v2.0/|)tenants$',
         services.GetTenants.as_view()),
     url(r'^(?P<servicetype>[0-9a-zA-Z_-]+)/(?P<requri>[0-9a-zA-Z./_-]*)$',
diff --git a/newton/newton/proxy/views/identityV3.py b/newton/newton/proxy/views/identityV3.py
index 49a1c1b7..8f14f389 100644
--- a/newton/newton/proxy/views/identityV3.py
+++ b/newton/newton/proxy/views/identityV3.py
@@ -27,93 +27,11 @@ from rest_framework.views import APIView
 from newton.pub.config import config
 from newton.pub.exceptions import VimDriverNewtonException
 from newton.requests.views.util import VimDriverUtils
+from newton.proxy.views.proxy_utils import ProxyUtils
 
 logger = logging.getLogger(__name__)
 
 DEBUG=True
-#MULTICLOUD_PREFIX = "http://%s:%s/api/multicloud-newton/v0" %(config.MSB_SERVICE_IP, config.MSB_SERVICE_PORT)
-
-def update_catalog(vimid, catalog, multicould_namespace):
-    '''
-    replace the orignal endpoints with multicloud's
-    return the catalog with updated endpoints, and also another catalog with prefix and suffix of each endpoint
-    :param vimid:
-    :param catalog: service catalog to be updated
-    :param multicould_namespace: multicloud namespace prefix to replace the real one in catalog endpoints url
-    :return:updated catalog, and metadata_catalog looks like:
-    {
-        'compute': {
-            'prefix': 'http://ip:port',
-            'proxy_prefix': 'http://another_ip: another_port',
-            'suffix': 'v2.1/53a4ab9015c84ee892e46d294f3b8b2d',
-        },
-        'network': {
-            'prefix': 'http://ip:port',
-            'proxy_prefix': 'http://another_ip: another_port',
-            'suffix': '',
-        },
-    }
-    '''
-
-    metadata_catalog = {}
-    if catalog:
-        # filter and replace endpoints of catalogs
-        for item in catalog:
-            one_catalog = {}
-            metadata_catalog[item['type']] = one_catalog
-
-            endpoints = item['endpoints']
-            item['endpoints']=[]
-            for endpoint in endpoints:
-                interface = endpoint.get('interface', None)
-                if interface != 'public':
-                    continue
-#                elif item["type"] == "identity":
-#                    endpoint["url"] = multicould_namespace + "/%s/identity/v3" % vimid
-                else:
-                    # replace the endpoint with MultiCloud's proxy
-                    import re
-                    endpoint_url = endpoint["url"]
-                    real_prefix = None
-                    real_suffix = None
-                    m = re.search(r'^(http[s]?://[0-9.]+:[0-9]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
-                    if not m:
-                        m = re.search(r'^(http[s]?://[0-9.]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
-                    if m:
-                        real_prefix = m.group(1)
-                        real_suffix = m.group(3)
-
-                    if real_prefix:
-                        # populate metadata_catalog
-                        one_catalog['prefix'] = real_prefix
-                        one_catalog['suffix'] = real_suffix if real_suffix else ''
-                        one_catalog['proxy_prefix'] = multicould_namespace + "/%s" % vimid + "/" + item["type"]
-
-                        endpoint_url = multicould_namespace + "/%s" % vimid + "/" + item["type"]
-
-                        if real_suffix:
-                            endpoint_url += "/" + real_suffix
-
-                        if item["type"] == "identity":
-                            endpoint_url = multicould_namespace + "/%s/identity/v3" % vimid
-
-#                        endpoint["url"] = re.sub(r"^http([s]*)://([0-9.]+):([0-9]+)",
-#                                                 multicould_namespace + "/%s/" % vimid + item["type"],
-#                                                 endpoint["url"])
-                    else:
-                        #something wrong
-                        pass
-
-
-                    endpoint["url"] = endpoint_url
-                item['endpoints'].append( endpoint )
-
-        return catalog, metadata_catalog
-    else:
-        return None
-
-
-
 
 class Tokens(APIView):
     service = {'service_type': 'identity',
@@ -124,7 +42,9 @@ class Tokens(APIView):
         self._logger = logger
 
     def post(self, request, vimid=""):
-        self._logger.debug("identityV3--post::> %s" % request.data)
+        self._logger.debug("identityV3--post::META> %s" % request.META)
+        self._logger.debug("identityV3--post::data> %s" % request.data)
+        self._logger.debug("identityV3--post::vimid> %s" % (vimid))
         sess = None
         resp = None
         resp_body = None
@@ -142,7 +62,7 @@ class Tokens(APIView):
             #set expiring in 1 hour
 
             #update the catalog
-            tmp_auth_data['token']['catalog'], tmp_metadata_catalog = update_catalog(vimid, tmp_auth_data['token']['catalog'], self.proxy_prefix)
+            tmp_auth_data['token']['catalog'], tmp_metadata_catalog = ProxyUtils.update_catalog(vimid, tmp_auth_data['token']['catalog'], self.proxy_prefix)
             tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state, json.dumps(tmp_metadata_catalog))
 
             resp = Response(headers={'X-Subject-Token': tmp_auth_token}, data=tmp_auth_data, status=status.HTTP_201_CREATED)
@@ -159,50 +79,3 @@ class Tokens(APIView):
             return Response(data={'error': str(e)},
                             status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-
-class Catalog(APIView):
-
-    service = {'service_type': 'identity',
-               'interface': 'public'}
-
-    def __init__(self):
-        self.proxy_prefix = config.MULTICLOUD_PREFIX
-        self._logger = logger
-
-    def get(self, request, vimid=""):
-        self._logger.debug("Catalog--get::data> %s" % request.data)
-#        self._logger.debug("Catalog--get::META> %s" % request.META)
-        try:
-            # prepare request resource to vim instance
-            #get token:
-            tmp_auth_token = request.META.get('HTTP_X_AUTH_TOKEN', None)
-            if not tmp_auth_token:
-                return Response(data={'error': "No X-Auth-Token found in headers"}, status=status.HTTP_401_UNAUTHORIZED)
-
-            vim = VimDriverUtils.get_vim_info(vimid)
-            #fetch the auth_state out of cache
-            tmp_auth_state, metadata_catalog = VimDriverUtils.get_token_cache(vim, tmp_auth_token)
-            if not tmp_auth_state:
-                return Response(data={'error': "Expired X-Auth-Token found in headers"}, status=status.HTTP_401_UNAUTHORIZED)
-
-            sess = VimDriverUtils.get_session(vim, auth_state=tmp_auth_state)
-            req_resource = "/auth/catalog"
-
-            resp = sess.get(req_resource, endpoint_filter=self.service)
-            #update token cache in case the token was required during the requests
-            tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state)
-
-            content = resp.json()
-            tmp_auth_catalog = content['catalog']
-            update_catalog(vimid, tmp_auth_catalog, self.proxy_prefix)
-
-            return Response(headers={'X-Subject-Token':tmp_auth_token}, data={'catalog': tmp_auth_catalog}, status=resp.status_code)
-        except VimDriverNewtonException as e:
-            return Response(data={'error': e.content}, status=e.status_code)
-        except HttpError as e:
-            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
-            return Response(data=e.response.json(), status=e.http_status)
-        except Exception as e:
-            self._logger.error(traceback.format_exc())
-            return Response(data={'error': str(e)},
-                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/newton/newton/proxy/views/proxy_utils.py b/newton/newton/proxy/views/proxy_utils.py
new file mode 100644
index 00000000..89a49286
--- /dev/null
+++ b/newton/newton/proxy/views/proxy_utils.py
@@ -0,0 +1,131 @@
+# Copyright (c) 2017 Wind River Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import json
+import traceback
+import re
+from rest_framework import status
+
+from newton.pub.exceptions import VimDriverNewtonException
+
+logger = logging.getLogger(__name__)
+
+DEBUG=True
+#MULTICLOUD_PREFIX = "http://%s:%s/api/multicloud-newton/v0" %(config.MSB_SERVICE_IP, config.MSB_SERVICE_PORT)
+
+class ProxyUtils(object):
+
+    @staticmethod
+    def update_prefix(metadata_catalog, content):
+        '''match the longgest prefix and replace it'''
+
+        if not content:
+            return content
+
+        for (servicetype, service_metadata) in metadata_catalog.items():
+            real_prefix = service_metadata['prefix']
+            proxy_prefix = service_metadata['proxy_prefix']
+
+            if content:
+                # filter the resp content and replace all endpoint prefix
+                tmp_content = json.dumps(content)
+                tmp_pattern = re.compile(real_prefix+r'([^:])')
+                tmp_content = tmp_pattern.sub(proxy_prefix+r'\1', tmp_content)
+                content = json.loads(tmp_content)
+
+        return content
+
+    @staticmethod
+    def update_catalog(vimid, catalog, multicould_namespace):
+        '''
+        replace the orignal endpoints with multicloud's
+        return the catalog with updated endpoints, and also another catalog with prefix and suffix of each endpoint
+        :param vimid:
+        :param catalog: service catalog to be updated
+        :param multicould_namespace: multicloud namespace prefix to replace the real one in catalog endpoints url
+        :return:updated catalog, and metadata_catalog looks like:
+        {
+            'compute': {
+                'prefix': 'http://ip:port',
+                'proxy_prefix': 'http://another_ip: another_port',
+                'suffix': 'v2.1/53a4ab9015c84ee892e46d294f3b8b2d',
+            },
+            'network': {
+                'prefix': 'http://ip:port',
+                'proxy_prefix': 'http://another_ip: another_port',
+                'suffix': '',
+            },
+        }
+        '''
+
+        metadata_catalog = {}
+        if catalog:
+            # filter and replace endpoints of catalogs
+            for item in catalog:
+                one_catalog = {}
+                metadata_catalog[item['type']] = one_catalog
+
+                endpoints = item['endpoints']
+                item['endpoints']=[]
+                for endpoint in endpoints:
+                    interface = endpoint.get('interface', None)
+                    if interface != 'public':
+                        continue
+    #                elif item["type"] == "identity":
+    #                    endpoint["url"] = multicould_namespace + "/%s/identity/v3" % vimid
+                    else:
+                        # replace the endpoint with MultiCloud's proxy
+                        import re
+                        endpoint_url = endpoint["url"]
+                        real_prefix = None
+                        real_suffix = None
+                        m = re.search(r'^(http[s]?://[0-9.]+:[0-9]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
+                        if not m:
+                            m = re.search(r'^(http[s]?://[0-9.]+)(/([0-9a-zA-Z/._-]+)$)?', endpoint_url)
+                        if m:
+                            real_prefix = m.group(1)
+                            real_suffix = m.group(3)
+
+                        if real_prefix:
+                            # populate metadata_catalog
+                            one_catalog['prefix'] = real_prefix
+                            one_catalog['suffix'] = real_suffix if real_suffix else ''
+                            one_catalog['proxy_prefix'] = multicould_namespace + "/%s" % vimid
+
+                            endpoint_url = multicould_namespace + "/%s" % vimid
+
+                            tmp_pattern = re.compile(item["type"])
+                            if not real_suffix or not re.match(tmp_pattern, real_suffix):
+                                one_catalog['proxy_prefix'] += "/" + item["type"]
+                                endpoint_url += '/' + item["type"]
+
+                            if real_suffix:
+                                endpoint_url += "/" + real_suffix
+
+                            if item["type"] == "identity":
+                                endpoint_url = multicould_namespace + "/%s/identity/v3" % vimid
+
+                        else:
+                            #something wrong
+                            pass
+
+                        endpoint["url"] = endpoint_url
+                    item['endpoints'].append( endpoint )
+
+            return catalog, metadata_catalog
+        else:
+            return None
+
+
+
diff --git a/newton/newton/proxy/views/services.py b/newton/newton/proxy/views/services.py
index 85de3aa6..1143464c 100644
--- a/newton/newton/proxy/views/services.py
+++ b/newton/newton/proxy/views/services.py
@@ -28,6 +28,7 @@ from rest_framework.views import APIView
 from newton.pub.exceptions import VimDriverNewtonException
 from newton.requests.views.util import VimDriverUtils
 from newton.pub.msapi import extsys
+from newton.proxy.views.proxy_utils import ProxyUtils
 
 logger = logging.getLogger(__name__)
 
@@ -144,13 +145,7 @@ class Services(APIView):
             #update token cache in case the token was required during the requests
             tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state)
             content = resp.json() if resp.content else None
-            if content:
-                #filter the resp content and replace all endpoint prefix
-                tmp_content = json.dumps(content)
-                tmp_pattern = re.compile(real_prefix)
-                tmp_content = tmp_pattern.sub(proxy_prefix, tmp_content)
-                content = json.loads(tmp_content)
-
+            content = ProxyUtils.update_prefix(metadata_catalog, content)
             return Response(headers={'X-Subject-Token': tmp_auth_token}, data=content, status=resp.status_code)
             #return resp
         except VimDriverNewtonException as e:
@@ -222,13 +217,7 @@ class Services(APIView):
             # update token cache in case the token was required during the requests
             tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state)
             content = resp.json() if resp.content else None
-            if content:
-                #filter the resp content and replace all endpoint prefix
-                tmp_content = json.dumps(content)
-                tmp_pattern = re.compile(real_prefix)
-                tmp_content = tmp_pattern.sub(proxy_prefix, tmp_content)
-                content = json.loads(tmp_content)
-
+            content = ProxyUtils.update_prefix(metadata_catalog, content)
             return Response(headers={'X-Subject-Token': tmp_auth_token}, data=content, status=resp.status_code)
 
         except VimDriverNewtonException as e:
@@ -300,13 +289,7 @@ class Services(APIView):
             # update token cache in case the token was required during the requests
             tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state)
             content = resp.json() if resp.content else None
-            if content:
-                #filter the resp content and replace all endpoint prefix
-                tmp_content = json.dumps(content)
-                tmp_pattern = re.compile(real_prefix)
-                tmp_content = tmp_pattern.sub(proxy_prefix, tmp_content)
-                content = json.loads(tmp_content)
-
+            content = ProxyUtils.update_prefix(metadata_catalog, content)
             return Response(headers={'X-Subject-Token': tmp_auth_token}, data=content, status=resp.status_code)
 
         except VimDriverNewtonException as e:
@@ -379,13 +362,7 @@ class Services(APIView):
             # update token cache in case the token was required during the requests
             tmp_auth_token = VimDriverUtils.update_token_cache(vim, sess, tmp_auth_token, tmp_auth_state)
             content = resp.json() if resp.content else None
-            if content:
-                #filter the resp content and replace all endpoint prefix
-                tmp_content = json.dumps(content)
-                tmp_pattern = re.compile(real_prefix)
-                tmp_content = tmp_pattern.sub(proxy_prefix, tmp_content)
-                content = json.loads(tmp_content)
-
+            content = ProxyUtils.update_prefix(metadata_catalog, content)
             return Response(headers={'X-Subject-Token': tmp_auth_token}, data=content, status=resp.status_code)
 
         except VimDriverNewtonException as e:
diff --git a/newton/newton/registration/views/registration.py b/newton/newton/registration/views/registration.py
index 131a58e0..25231d7c 100644
--- a/newton/newton/registration/views/registration.py
+++ b/newton/newton/registration/views/registration.py
@@ -33,9 +33,9 @@ DEBUG=True
 class Registry(APIView):
 
     def __init__(self):
+        self.proxy_prefix = config.MULTICLOUD_PREFIX
         self._logger = logger
 
-
     def update_tenant(self, cloud_owner, cloud_region_id, tenantinfo):
         '''
         populate tenant into AAI
@@ -668,7 +668,7 @@ class Registry(APIView):
             # populate proxy identity url
             cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid)
             self.update_proxy_identity_endpoint(cloud_owner, cloud_region_id,
-                                                config.MULTICLOUD_PREFIX + "/%s/identity/v3" % vimid)
+                                                self.proxy_prefix + "/%s/identity/v3" % vimid)
 
             # prepare request resource to vim instance
             # get token: