openstak newton version vim driver implementation

1. change to use keystoneauth1 for auth and transport
2. add neutron network creation and deletion handling
3. some modification for refactoring

Change-Id: Ibaa3f33ae89c57bcb5a5e603b63355ed7ca4c52b
Issue-Id: MULTIVIM-47
Signed-off-by: Litao Gao <litao.gao@windriver.com>

8 files changed, 247 insertions, 183 deletions

diff --git a/newton/newton/pub/msapi/extsys.py b/newton/newton/pub/msapi/extsys.py
index 297d4716..daab5037 100644
--- a/newton/newton/pub/msapi/extsys.py
+++ b/newton/newton/pub/msapi/extsys.py
@@ -12,6 +12,7 @@
 import json
 import logging
 
+from rest_framework import status
 from newton.pub.exceptions import VimDriverNewtonException
 from newton.pub.utils.restcall import req_by_msb
 
@@ -33,5 +34,6 @@ def get_vim_by_id(vim_id):
     if retcode != 0:
         logger.error("Status code is %s, detail is %s.", status_code, content)
         raise VimDriverNewtonException(
-            "Failed to query VIM with id (%s) from extsys." % vim_id)
+            "Failed to query VIM with id (%s) from extsys." % vim_id,
+            status.HTTP_404_NOT_FOUND, content)
     return json.JSONDecoder().decode(content)
diff --git a/newton/newton/pub/utils/restcall.py b/newton/newton/pub/utils/restcall.py
index 7f372982..ccbe1014 100644
--- a/newton/newton/pub/utils/restcall.py
+++ b/newton/newton/pub/utils/restcall.py
@@ -14,8 +14,11 @@ import traceback
 import logging
 import urllib2
 import uuid
+import httplib
 import httplib2
 
+from rest_framework import status
+
 from newton.pub.config.config import MSB_SERVICE_IP, MSB_SERVICE_PORT
 
 rest_no_auth, rest_oneway_auth, rest_bothway_auth = 0, 1, 2
@@ -36,7 +39,7 @@ def call_req(base_url, user, passwd, auth_type,
     logger.debug("[%s]call_req('%s','%s','%s',%s,'%s','%s','%s')" % (
         callid, base_url, user, passwd, auth_type, resource, method, content))
     ret = None
-    resp_status = ''
+    resp_status = None
     try:
         full_url = combine_url(base_url, resource)
         headers = {
@@ -69,26 +72,18 @@ def call_req(base_url, user, passwd, auth_type,
                 else:
                     ret = [1, resp_body, resp_status]
                 break
-            except Exception as ex:
-                if 'httplib.ResponseNotReady' in str(sys.exc_info()):
-                    logger.debug("retry_times=%d", retry_times)
-                    logger.error(traceback.format_exc())
-                    ret = [1, "Unable to connect to %s" % full_url, resp_status]
-                    continue
-                raise ex
+            except httplib.ResponseNotReady:
+                logger.debug("retry_times=%d", retry_times)
+                ret = [1, "Unable to connect to %s" % full_url, resp_status]
+                continue
     except urllib2.URLError as err:
         ret = [2, str(err), resp_status]
-    except Exception as ex:
+    except Exception:
         logger.error(traceback.format_exc())
         logger.error("[%s]ret=%s" % (callid, str(sys.exc_info())))
-        res_info = str(sys.exc_info())
-        if 'httplib.ResponseNotReady' in res_info:
-            res_info = ("The URL[%s] request failed or is not responding." %
-                        full_url)
-        ret = [3, res_info, resp_status]
-    except:
-        logger.error(traceback.format_exc())
-        ret = [4, str(sys.exc_info()), resp_status]
+        if not resp_status:
+            resp_status = status.HTTP_500_INTERNAL_SERVER_ERROR
+        ret = [3, str(sys.exc_info()), resp_status]
 
     logger.debug("[%s]ret=%s" % (callid, str(ret)))
     return ret
diff --git a/newton/newton/requests/urls.py b/newton/newton/requests/urls.py
index f832eec4..e90d9ab6 100644
--- a/newton/newton/requests/urls.py
+++ b/newton/newton/requests/urls.py
@@ -15,11 +15,13 @@
 from django.conf.urls import url
 from rest_framework.urlpatterns import format_suffix_patterns
 
-from . import views
+from views import network
 
 urlpatterns = [
-    url(r'^networks(/(?P<networkid>[0-9a-zA-Z_-]+))?', views.Networks.as_view()),
-    url(r'^subnets/(?P<subnetid>[0-9a-zA-Z_-]+)', views.Subnets.as_view()),
+    url(r'^networks(/(?P<networkid>[0-9a-zA-Z_-]+))?',
+        network.Networks.as_view()),
+    url(r'^subnets/(?P<subnetid>[0-9a-zA-Z_-]+)',
+        network.Subnets.as_view()),
 ]
 
 urlpatterns = format_suffix_patterns(urlpatterns)
diff --git a/newton/newton/requests/views.py b/newton/newton/requests/views.py
deleted file mode 100644
index 0333f86c..00000000
--- a/newton/newton/requests/views.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# Copyright (c) 2017 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-import json
-from json import JSONEncoder
-
-from rest_framework import status
-from rest_framework.response import Response
-from rest_framework.views import APIView
-
-from newton.pub.utils.restcall import req_to_vim
-from newton.pub.exceptions import VimDriverNewtonException
-from newton.pub.msapi.extsys import get_vim_by_id
-
-logger = logging.getLogger(__name__)
-
-
-class VimDriverUtils(object):
-    @staticmethod
-    def get_vim_info(vimid):
-        # get vim info from local cache firstly
-        # if cache miss, get it from ESR service
-        vim = get_vim_by_id(vimid)
-        return vim
-
-    @staticmethod
-    def relay_request_to_vim_service(vimid, tenantid, request, service_type,
-                                     req_resouce, req_content=""):
-        """
-        if there is no token cache, do auth
-        if there is, use token directly
-        if response is 'need auth', do auth
-        get the use extractor to get token
-        and service rul and then do request
-        """
-        vim = VimDriverUtils.get_vim_info(vimid)
-        auth_resouce = "/tokens"
-        method = "POST"
-        headers = ""
-        r_content_dict = {
-            "auth": {
-                "tenantName": vim["tenant"],
-                "passwordCredentials": {
-                    "username": vim["userName"],
-                    "password": vim["password"]
-                }
-            }
-        }
-        r_content = JSONEncoder().encode(r_content_dict)
-        retcode, content, status_code = \
-            req_to_vim(vim["url"], auth_resouce, method, headers, r_content)
-        if retcode != 0:
-            logger.error("Status code is %s, detail is %s.",
-                         status_code, content)
-            raise VimDriverNewtonException("Fail to authorize",
-                                           status_code, content)
-        else:
-            # extract token id and issue the get request
-            json_content = None
-            auth_resp = json.JSONDecoder().decode(content)
-            tokenid, svcurl = VimDriverUtils.extractor(auth_resp, service_type)
-            method = request.method
-            headers = {'X-Auth-Token': tokenid}
-            retcode, content, status_code = \
-                req_to_vim(svcurl, req_resouce, method, headers, req_content)
-            if retcode != 0:
-                logger.error("Status code is %s, detail is %s.",
-                             status_code, content)
-                raise VimDriverNewtonException("Fail to complte request",
-                                               status_code, content)
-            else:
-                json_content = json.JSONDecoder().decode(content)
-                vim_dict = {
-                    "vimName": vim["name"],
-                    "vimId": vim["vimId"],
-                    "tenantId": tenantid,
-                }
-                json_content.update(vim_dict)
-            return status_code, json_content
-
-    @staticmethod
-    def extractor(resp_data, service_type):
-        try:
-            tokenid = resp_data["access"]["token"]["id"]
-            sc = resp_data["access"]["serviceCatalog"]
-            service = [svc for svc in sc if svc["type"] == service_type]
-            return tokenid, service[0]["endpoints"][0]["publicURL"]
-        except Exception:
-            raise Exception(
-                "There is no valid %s token or service info" % service_type)
-
-
-class Networks(APIView):
-    SERVICE = "network"
-    keys_map_resp = [
-        ("provider:segmentation_id", "segmentationId"),
-        ("provider:physical_network", "physicalNetwork"),
-        ("router:external", "routerExternal"),
-        ("provider:network_type", "networkType"),
-        ("vlan_transparent", "vlanTransparent"),
-    ]
-
-    def get(self, request, vimid="", tenantid="", networkid=""):
-        logger.debug("Networks--get::> %s" % request.data)
-        try:
-            # prepare request resource to vim instance
-            req_resouce = "v2.0/networks"
-            full_path = request.get_full_path()
-            if '?' in full_path:
-                _, query = request.get_full_path().split('?')
-                req_resouce += "?%s" % query
-            status_code, content = VimDriverUtils.relay_request_to_vim_service(
-                vimid, tenantid, request, self.SERVICE, req_resouce)
-
-            logger.debug("response content after11: %s" % content)
-            # convert the key naming in networks
-            for network in content["networks"]:
-                for k in self.keys_map_resp:
-                    v = network.pop(k[0], None)
-                    if v:
-                        network[k[1]] = v
-            logger.debug("response content after: %s" % content)
-            return Response(data=content, status=status_code)
-        except VimDriverNewtonException as e:
-            return Response(data=e.content, status=e.status_code)
-        except Exception as e:
-            return Response(data={'error': e},
-                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
-    def post(self, request, tenantid=""):
-        logger.debug("Networks--post::> %s" % request.data)
-        pass
-
-    def delete(self, request):
-        logger.debug("Networks--delete::> %s" % request.data)
-        pass
-
-
-class Subnets(APIView):
-    pass
diff --git a/newton/newton/requests/views/__init__.py b/newton/newton/requests/views/__init__.py
new file mode 100644
index 00000000..802f3fba
--- /dev/null
+++ b/newton/newton/requests/views/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) 2017 Wind River Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/newton/newton/requests/views/network.py b/newton/newton/requests/views/network.py
new file mode 100644
index 00000000..9d91afc3
--- /dev/null
+++ b/newton/newton/requests/views/network.py
@@ -0,0 +1,139 @@
+# Copyright (c) 2017 Wind River Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import json
+
+from rest_framework import status
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from newton.pub.exceptions import VimDriverNewtonException
+
+from util import VimDriverUtils
+
+logger = logging.getLogger(__name__)
+
+
+class Networks(APIView):
+    service = {'service_type': 'network',
+               'interface': 'public',
+               'region_name': 'RegionOne'}
+    keys_mapping = [
+        ("provider:segmentation_id", "segmentationId"),
+        ("provider:physical_network", "physicalNetwork"),
+        ("router:external", "routerExternal"),
+        ("provider:network_type", "networkType"),
+        ("vlan_transparent", "vlanTransparent"),
+        ("project_id", "tenantId"),
+    ]
+
+    def get(self, request, vimid="", tenantid="", networkid=""):
+        logger.debug("Networks--get::> %s" % request.data)
+        try:
+            # prepare request resource to vim instance
+            req_resouce = "v2.0/networks"
+            if networkid:
+                req_resouce += "/%s" % networkid
+            query = VimDriverUtils.get_query_part(request)
+            if query:
+                req_resouce += "?%s" % query
+
+            vim = VimDriverUtils.get_vim_info(vimid)
+            sess = VimDriverUtils.get_session(vim, tenantid)
+            resp = sess.get(req_resouce, endpoint_filter=self.service)
+            content = resp.json()
+            vim_dict = {
+                "vimName": vim["name"],
+                "vimId": vim["vimId"],
+                "tenantId": tenantid,
+            }
+            content.update(vim_dict)
+
+            if not networkid:
+                # convert the key naming in networks
+                for network in content["networks"]:
+                    VimDriverUtils.replace_key_by_mapping(network,
+                                                          self.keys_mapping)
+            else:
+                # convert the key naming in the network specified by id
+                VimDriverUtils.replace_key_by_mapping(content["network"],
+                                                      self.keys_mapping)
+
+            return Response(data=content, status=resp.status_code)
+        except VimDriverNewtonException as e:
+            return Response(data={'error': e.content}, status=e.status_code)
+        except Exception as e:
+            return Response(data={'error': str(e)},
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+    def post(self, request, vimid="", tenantid="", networkid=""):
+        logger.debug("Networks--post::> %s" % request.data)
+        try:
+            # prepare request resource to vim instance
+            req_resouce = "v2.0/networks"
+            if networkid:
+                req_resouce += "/%s" % networkid
+            query = VimDriverUtils.get_query_part(request)
+            if query:
+                req_resouce += "?%s" % query
+
+            vim = VimDriverUtils.get_vim_info(vimid)
+            sess = VimDriverUtils.get_session(vim, tenantid)
+            network = request.data
+            VimDriverUtils.replace_key_by_mapping(network,
+                                                  self.keys_mapping, True)
+            req_body = json.JSONEncoder().encode({"network": network})
+            resp = sess.post(req_resouce, data=req_body,
+                             endpoint_filter=self.service)
+            resp_body = resp.json()["network"]
+            VimDriverUtils.replace_key_by_mapping(resp_body, self.keys_mapping)
+            vim_dict = {
+                "vimName": vim["name"],
+                "vimId": vim["vimId"],
+                "tenantId": tenantid,
+            }
+            resp_body.update(vim_dict)
+            return Response(data=resp_body, status=resp.status_code)
+        except VimDriverNewtonException as e:
+            return Response(data={'error': e.content}, status=e.status_code)
+        except Exception as e:
+            return Response(data={'error': str(e)},
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        pass
+
+    def delete(self, request, vimid="", tenantid="", networkid=""):
+        logger.debug("Networks--delete::> %s" % request.data)
+        try:
+            # prepare request resource to vim instance
+            req_resouce = "v2.0/networks"
+            if networkid:
+                req_resouce += "/%s" % networkid
+            query = VimDriverUtils.get_query_part(request)
+            if query:
+                req_resouce += "?%s" % query
+
+            vim = VimDriverUtils.get_vim_info(vimid)
+            sess = VimDriverUtils.get_session(vim, tenantid)
+            resp = sess.delete(req_resouce, endpoint_filter=self.service)
+            return Response(status=resp.status_code)
+        except VimDriverNewtonException as e:
+            return Response(data={'error': e.content}, status=e.status_code)
+        except Exception as e:
+            return Response(data={'error': str(e)},
+                            status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+        pass
+
+
+class Subnets(APIView):
+    pass
diff --git a/newton/newton/requests/views/util.py b/newton/newton/requests/views/util.py
new file mode 100644
index 00000000..10a6389b
--- /dev/null
+++ b/newton/newton/requests/views/util.py
@@ -0,0 +1,76 @@
+# Copyright (c) 2017 Wind River Systems, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+from keystoneauth1.identity import v2 as keystone_v2
+from keystoneauth1.identity import v3 as keystone_v3
+from keystoneauth1 import session
+
+from newton.pub.msapi.extsys import get_vim_by_id
+
+logger = logging.getLogger(__name__)
+
+
+class VimDriverUtils(object):
+    @staticmethod
+    def get_vim_info(vimid):
+        # get vim info from local cache firstly
+        # if cache miss, get it from ESR service
+        vim = get_vim_by_id(vimid)
+        return vim
+
+    @staticmethod
+    def get_query_part(request):
+        query = ""
+        full_path = request.get_full_path()
+        if '?' in full_path:
+            _, query = request.get_full_path().split('?')
+        return query
+
+    @staticmethod
+    def get_session(vim, tenantid=None):
+        """
+        get vim info from ESR and create auth plugin and session object
+        """
+        auth = None
+        if '/v2' in vim["url"]:
+            auth = keystone_v2.Password(auth_url=vim["url"],
+                                        username=vim["userName"],
+                                        password=vim["password"],
+                                        tenant_name=vim["tenant"])
+        elif '/v3' in vim["url"]:
+            auth = keystone_v3.Password(auth_url=vim["url"],
+                                        username=vim["userName"],
+                                        password=vim["password"],
+                                        project_name=vim["tenant"],
+                                        user_domain_id='default',
+                                        project_domain_id='default')
+        return session.Session(auth=auth)
+
+    @staticmethod
+    def replace_a_key(dict_obj, keypair, reverse=False):
+        old_key, new_key = None, None
+        if reverse:
+            old_key, new_key = keypair[1], keypair[0]
+        else:
+            old_key, new_key = keypair[0], keypair[1]
+
+        v = dict_obj.pop(old_key, None)
+        if v:
+            dict_obj[new_key] = v
+
+    @staticmethod
+    def replace_key_by_mapping(dict_obj, mapping, reverse=False):
+        for k in mapping:
+            VimDriverUtils.replace_a_key(dict_obj, k)
diff --git a/newton/requirements.txt b/newton/requirements.txt
index 6d58957b..5976f5c9 100644
--- a/newton/requirements.txt
+++ b/newton/requirements.txt
@@ -2,20 +2,11 @@
 Django==1.9.6
 djangorestframework==3.3.3
 
-# redis cache
-redis==2.10.5
-
-# for access redis cache
-redisco==0.1.4
-django-redis-cache==0.13.1
-
 # for call rest api
 httplib2==0.9.2
 
-# for call openstack api
-python-keystoneclient==3.6.0
-python-glanceclient==2.5.0
-python-neutronclient==6.0.0
+# for call openstack auth and transport api
+keystoneauth1==2.18.0
 
 # for unit test
 coverage==4.2