From 5eecba06f3173ec8d163e34bff287a2b09e5829c Mon Sep 17 00:00:00 2001 From: Bin Yang Date: Thu, 28 Mar 2019 13:17:14 +0000 Subject: Initiate a thread to handle registry To prevent the registry API timeout by offloading the time-consuming registry to a thread Change-Id: I1107bb7f3d9a11bdae4d0a5926f788bc2a534bda Issue-ID: MULTICLOUD-554 Signed-off-by: Bin Yang --- .../registration/tests/test_registration2.py | 12 +- .../thinkcloud/registration/views/registration.py | 2 + newton/newton/registration/views/registration.py | 1 + .../ocata/registration/tests/test_registration2.py | 12 +- ocata/ocata/registration/views/registration.py | 2 + pike/pike/registration/views/registration.py | 2 + share/newton_base/registration/registration.py | 571 +++++++++++++-------- share/starlingx_base/registration/registration.py | 269 +++++----- .../registration/tests/test_registration2.py | 12 +- .../registration/tests/test_registration2.py | 8 +- 10 files changed, 513 insertions(+), 378 deletions(-) diff --git a/lenovo/thinkcloud/registration/tests/test_registration2.py b/lenovo/thinkcloud/registration/tests/test_registration2.py index d799c90a..7abd6073 100644 --- a/lenovo/thinkcloud/registration/tests/test_registration2.py +++ b/lenovo/thinkcloud/registration/tests/test_registration2.py @@ -92,8 +92,10 @@ class TestRegistration2(unittest.TestCase): } ) - resp = self.view._discover_flavors(vimid="lenovo-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="lenovo-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) @@ -110,7 +112,9 @@ class TestRegistration2(unittest.TestCase): } ), - resp = self.view._discover_flavors(vimid="lenovo-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="lenovo-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) diff --git a/lenovo/thinkcloud/registration/views/registration.py b/lenovo/thinkcloud/registration/views/registration.py index 0fe4a80e..c55bf712 100644 --- a/lenovo/thinkcloud/registration/views/registration.py +++ b/lenovo/thinkcloud/registration/views/registration.py @@ -29,6 +29,7 @@ class Registry(newton_registration.Registry): self.proxy_prefix = settings.MULTICLOUD_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(Registry, self).__init__() class RegistryV1(Registry): @@ -36,6 +37,7 @@ class RegistryV1(Registry): self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(RegistryV1, self).__init__() def post(self, request, cloud_owner="", cloud_region_id=""): self._logger.info("registration with : %s, %s" % (cloud_owner, cloud_region_id)) diff --git a/newton/newton/registration/views/registration.py b/newton/newton/registration/views/registration.py index ea160cd0..313f3acf 100644 --- a/newton/newton/registration/views/registration.py +++ b/newton/newton/registration/views/registration.py @@ -34,3 +34,4 @@ class Registry(newton_registration.Registry): self.proxy_prefix = settings.MULTICLOUD_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(Registry, self).__init__() diff --git a/ocata/ocata/registration/tests/test_registration2.py b/ocata/ocata/registration/tests/test_registration2.py index 757dccc0..bfb7d7a5 100644 --- a/ocata/ocata/registration/tests/test_registration2.py +++ b/ocata/ocata/registration/tests/test_registration2.py @@ -92,8 +92,10 @@ class TestRegistration2(unittest.TestCase): } ) - resp = self.view._discover_flavors(vimid="windriver-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="windriver-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) @@ -110,7 +112,9 @@ class TestRegistration2(unittest.TestCase): } ), - resp = self.view._discover_flavors(vimid="windriver-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="windriver-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) diff --git a/ocata/ocata/registration/views/registration.py b/ocata/ocata/registration/views/registration.py index 0de84d68..d54ea852 100644 --- a/ocata/ocata/registration/views/registration.py +++ b/ocata/ocata/registration/views/registration.py @@ -29,6 +29,7 @@ class Registry(newton_registration.Registry): self.proxy_prefix = settings.MULTICLOUD_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(Registry, self).__init__() class RegistryV1(Registry): @@ -36,6 +37,7 @@ class RegistryV1(Registry): self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(RegistryV1, self).__init__() def post(self, request, cloud_owner="", cloud_region_id=""): self._logger.info("registration with : %s, %s" % (cloud_owner, cloud_region_id)) diff --git a/pike/pike/registration/views/registration.py b/pike/pike/registration/views/registration.py index 2b34094e..a437e2a4 100644 --- a/pike/pike/registration/views/registration.py +++ b/pike/pike/registration/views/registration.py @@ -30,6 +30,7 @@ class Registry(newton_registration.Registry): self.proxy_prefix = settings.MULTICLOUD_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(Registry, self).__init__() class RegistryV1(Registry): @@ -37,6 +38,7 @@ class RegistryV1(Registry): self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + super(RegistryV1, self).__init__() def post(self, request, cloud_owner="", cloud_region_id=""): self._logger.info("registration with : %s, %s" % (cloud_owner, cloud_region_id)) diff --git a/share/newton_base/registration/registration.py b/share/newton_base/registration/registration.py index 35f3b982..9aef690a 100644 --- a/share/newton_base/registration/registration.py +++ b/share/newton_base/registration/registration.py @@ -16,6 +16,7 @@ import logging import json import uuid import traceback +import threading from keystoneauth1.exceptions import HttpError from rest_framework import status @@ -33,10 +34,289 @@ logger = logging.getLogger(__name__) class Registry(APIView): def __init__(self): - self.proxy_prefix = "multicloud" - self.aai_base_url = "127.0.0.1" + if not hasattr(self, "_logger"): + self._logger = logger + + if not hasattr(self, "register_helper") or not self.register_helper: + if not hasattr(self, "proxy_prefix"): + self.proxy_prefix = "multicloud" + if not hasattr(self, "AAI_BASE_URL"): + self.AAI_BASE_URL = "127.0.0.1" + self.register_helper = RegistryHelper(self.proxy_prefix or "multicloud", self.AAI_BASE_URL or "127.0.0.1") + + def post(self, request, vimid=""): + self._logger.info("registration with vimid: %s" % vimid) + self._logger.debug("with data: %s" % request.data) + + try: + + thread1 = RegisterHelperThread(self.register_helper.registry) + thread1.addv0(vimid) + if 0 == thread1.state(): + thread1.start() + + return Response(status=status.HTTP_202_ACCEPTED) + + except VimDriverNewtonException as e: + return Response(data={'error': e.content}, status=e.status_code) + except HttpError as e: + self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json())) + return Response(data=e.response.json(), status=e.http_status) + except Exception as e: + self._logger.error(traceback.format_exc()) + return Response( + data={'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def delete(self, request, vimid=""): + self._logger.debug("Registration--delete::data> %s" % request.data) + self._logger.debug("Registration--delete::vimid > %s"% vimid) + try: + + retcode = RegistryHelper.unregistry(vimid) + + #ret_code = VimDriverUtils.delete_vim_info(vimid) + return Response(status=status.HTTP_204_NO_CONTENT if retcode==0 else status.HTTP_500_INTERNAL_SERVER_ERROR) + except VimDriverNewtonException as e: + return Response(data={'error': e.content}, status=e.status_code) + except HttpError as e: + self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json())) + return Response(data=e.response.json(), status=e.http_status) + except Exception as e: + self._logger.error(traceback.format_exc()) + return Response(data={'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + +class RegistryHelper(object): + ''' + Helper code to discover and register a cloud region's resource + ''' + + def __init__(self, multicloud_prefix, aai_base_url): + self.proxy_prefix = multicloud_prefix + self.aai_base_url = aai_base_url self._logger = logger + def registry(self, vimid=""): + # populate proxy identity url + self._update_proxy_identity_endpoint(vimid) + + # prepare request resource to vim instance + # get token: + viminfo = VimDriverUtils.get_vim_info(vimid) + if not viminfo: + raise VimDriverNewtonException( + "There is no cloud-region with {cloud-owner}_{cloud-region-id}=%s in AAI" % vimid) + + # set the default tenant since there is no tenant info in the VIM yet + sess = VimDriverUtils.get_session( + viminfo, tenant_name=viminfo['tenant']) + + # step 1. discover all projects and populate into AAI + self._discover_tenants(vimid, sess, viminfo) + + # discover all flavors + self._discover_flavors(vimid, sess, viminfo) + + # discover all images + self._discover_images(vimid, sess, viminfo) + + # discover all az + self._discover_availability_zones(vimid, sess, viminfo) + + # discover all vg + #self._discover_volumegroups(vimid, sess, viminfo) + + # discover all snapshots + #self._discover_snapshots(vimid, sess, viminfo) + + # discover all server groups + #self.discover_servergroups(request, vimid, sess, viminfo) + + # discover all pservers + #self._discover_pservers(vimid, sess, viminfo) + + return 0 + + + def unregistry(self, vimid=""): + + # prepare request resource to vim instance + # get token: + viminfo = VimDriverUtils.get_vim_info(vimid) + if not viminfo: + raise VimDriverNewtonException( + "There is no cloud-region with {cloud-owner}_{cloud-region-id}=%s in AAI" % vimid) + + cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + + # get the resource first + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s?depth=all" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + }) + + # get cloud-region + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "GET") + + # add resource-version + if retcode == 0 and content: + cloudregiondata = json.JSONDecoder().decode(content) + + # step 1. remove all tenants + tenants = cloudregiondata.get("tenants", None) + for tenant in tenants.get("tenant", []) if tenants else []: + # common prefix + aai_cloud_region = "/cloud-infrastructure/cloud-regions/cloud-region/%s/%s/tenants/tenant/%s" \ + % (cloud_owner, cloud_region_id, tenant['tenant-id']) + + # remove all vservers + try: + # get list of vservers + vservers = tenant.get('vservers', {}).get('vserver', []) + for vserver in vservers: + try: + # iterate vport, except will be raised if no l-interface exist + for vport in vserver['l-interfaces']['l-interface']: + # delete vport + vport_delete_url = aai_cloud_region + "/vservers/vserver/%s/l-interfaces/l-interface/%s?resource-version=%s" \ + % (vserver['vserver-id'], vport['interface-name'], + vport['resource-version']) + restcall.req_to_aai(vport_delete_url, "DELETE") + except Exception as e: + pass + + try: + # delete vserver + vserver_delete_url = aai_cloud_region + "/vservers/vserver/%s?resource-version=%s" \ + % ( + vserver['vserver-id'], vserver['resource-version']) + restcall.req_to_aai(vserver_delete_url, "DELETE") + except Exception as e: + continue + + except Exception: + self._logger.error(traceback.format_exc()) + pass + + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" + "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource_type": "tenant", + "resoure_id": tenant["tenant-id"], + "resource-version": tenant["resource-version"] + }) + # remove tenant + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + # remove all flavors + flavors = cloudregiondata.get("flavors", None) + for flavor in flavors.get("flavor", []) if flavors else []: + # iterate hpa-capabilities + hpa_capabilities = flavor.get("hpa-capabilities", None) + for hpa_capability in hpa_capabilities.get("hpa-capability", []) if hpa_capabilities else []: + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" + "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" + "hpa-capabilities/hpa-capability/%(hpa-capability-id)s/" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource_type": "flavor", + "resoure_id": flavor["flavor-id"], + "hpa-capability-id": hpa_capability["hpa-capability-id"], + "resource-version": hpa_capability["resource-version"] + }) + # remove hpa-capability + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + # remove flavor + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" + "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource_type": "flavor", + "resoure_id": flavor["flavor-id"], + "resource-version": flavor["resource-version"] + }) + + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + # remove all images + images = cloudregiondata.get("images", None) + for image in images.get("image", []) if images else []: + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" + "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource_type": "image", + "resoure_id": image["image-id"], + "resource-version": image["resource-version"] + }) + # remove image + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + # remove all az + + # remove all vg + + # remove all snapshots + snapshots = cloudregiondata.get("snapshots", None) + for snapshot in snapshots.get("snapshot", []) if snapshots else []: + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" + "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource_type": "snapshot", + "resoure_id": snapshot["snapshot-id"], + "resource-version": snapshot["resource-version"] + }) + # remove snapshot + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + # remove all server groups + + # remove all pservers + + # remove cloud region itself + resource_url = ("/cloud-infrastructure/cloud-regions/" + "cloud-region/%(cloud_owner)s/%(cloud_region_id)s" + "?resource-version=%(resource-version)s" + % { + "cloud_owner": cloud_owner, + "cloud_region_id": cloud_region_id, + "resource-version": cloudregiondata["resource-version"] + }) + # remove cloud region + retcode, content, status_code = \ + restcall.req_to_aai(resource_url, "DELETE") + + return retcode, content, status_code + + def _get_list_resources( self, resource_url, service_type, session, viminfo, vimid, content_key): @@ -972,247 +1252,82 @@ class Registry(APIView): self._logger.error(traceback.format_exc()) return - def post(self, request, vimid=""): - self._logger.info("registration with vimid: %s" % vimid) - self._logger.debug("with data: %s" % request.data) - - try: - # populate proxy identity url - self._update_proxy_identity_endpoint(vimid) - - # prepare request resource to vim instance - # get token: - viminfo = VimDriverUtils.get_vim_info(vimid) - if not viminfo: - raise VimDriverNewtonException( - "There is no cloud-region with {cloud-owner}_{cloud-region-id}=%s in AAI" % vimid) - - # set the default tenant since there is no tenant info in the VIM yet - sess = VimDriverUtils.get_session( - viminfo, tenant_name=viminfo['tenant']) - # step 1. discover all projects and populate into AAI - self._discover_tenants(vimid, sess, viminfo) +class RegisterHelperThread(threading.Thread): + ''' + thread to register infrastructure resource into AAI + ''' - # discover all flavors - self._discover_flavors(vimid, sess, viminfo) + def __init__(self, registry_helper): + threading.Thread.__init__(self) + self.daemon = True + self.duration = 0 + self.helper = registry_helper - # discover all images - self._discover_images(vimid, sess, viminfo) + # The queue of IDs of cloud regions, format: + # v0: "owner1_regionid1" + self.queuev0 = [] - # discover all az - self._discover_availability_zones(vimid, sess, viminfo) + # v1: {"cloud-owner": "owner1", "cloud-region-id": "regionid1"}, + self.queuev1 = [] + self.lock = threading.Lock() - # discover all vg - #self._discover_volumegroups(vimid, sess, viminfo) + self.state_ = 0 # 0: stopped, 1: started - # discover all snapshots - #self._discover_snapshots(vimid, sess, viminfo) + def addv0(self, vimid): + self.lock.acquire() + self.queuev0.append(vimid) + self.lock.release() + return len(self.queuev0) - # discover all server groups - #self.discover_servergroups(request, vimid, sess, viminfo) - - # discover all pservers - #self._discover_pservers(vimid, sess, viminfo) - - return Response(status=status.HTTP_202_ACCEPTED) - - except VimDriverNewtonException as e: - return Response(data={'error': e.content}, status=e.status_code) - except HttpError as e: - self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json())) - return Response(data=e.response.json(), status=e.http_status) - except Exception as e: - self._logger.error(traceback.format_exc()) - return Response( - data={'error': str(e)}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR) - - def delete(self, request, vimid=""): - self._logger.debug("Registration--delete::data> %s" % request.data) - self._logger.debug("Registration--delete::vimid > %s"% vimid) - try: - - # prepare request resource to vim instance - # get token: - viminfo = VimDriverUtils.get_vim_info(vimid) - if not viminfo: - raise VimDriverNewtonException( - "There is no cloud-region with {cloud-owner}_{cloud-region-id}=%s in AAI" % vimid) - - cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) - - #get the resource first - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s?depth=all" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - }) - - # get cloud-region - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "GET") - - # add resource-version - if retcode == 0 and content: - cloudregiondata = json.JSONDecoder().decode(content) - - # step 1. remove all tenants - tenants = cloudregiondata.get("tenants", None) - for tenant in tenants.get("tenant", []) if tenants else []: - # common prefix - aai_cloud_region = "/cloud-infrastructure/cloud-regions/cloud-region/%s/%s/tenants/tenant/%s" \ - % (cloud_owner, cloud_region_id, tenant['tenant-id']) - - # remove all vservers - try: - # get list of vservers - vservers = tenant.get('vservers', {}).get('vserver', []) - for vserver in vservers: - try: - # iterate vport, except will be raised if no l-interface exist - for vport in vserver['l-interfaces']['l-interface']: - # delete vport - vport_delete_url = aai_cloud_region + "/vservers/vserver/%s/l-interfaces/l-interface/%s?resource-version=%s" \ - % (vserver['vserver-id'], vport['interface-name'], - vport['resource-version']) - restcall.req_to_aai(vport_delete_url, "DELETE") - except Exception as e: - pass - - try: - # delete vserver - vserver_delete_url = aai_cloud_region + "/vservers/vserver/%s?resource-version=%s" \ - % ( - vserver['vserver-id'], vserver['resource-version']) - restcall.req_to_aai(vserver_delete_url, "DELETE") - except Exception as e: - continue - - except Exception: - self._logger.error(traceback.format_exc()) - pass - - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" - "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource_type": "tenant", - "resoure_id": tenant["tenant-id"], - "resource-version": tenant["resource-version"] - }) - # remove tenant - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + def removev0(self, vimid): + ''' + remove cloud region from list + ''' + self.queuev0 = [x for x in self.queuev0 if x != vimid] - # remove all flavors - flavors = cloudregiondata.get("flavors", None) - for flavor in flavors.get("flavor", []) if flavors else []: - # iterate hpa-capabilities - hpa_capabilities = flavor.get("hpa-capabilities", None) - for hpa_capability in hpa_capabilities.get("hpa-capability", []) if hpa_capabilities else []: - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" - "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" - "hpa-capabilities/hpa-capability/%(hpa-capability-id)s/" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource_type": "flavor", - "resoure_id": flavor["flavor-id"], - "hpa-capability-id": hpa_capability["hpa-capability-id"], - "resource-version": hpa_capability["resource-version"] - }) - # remove hpa-capability - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + def resetv0(self): + self.queuev0 = [] - # remove flavor - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" - "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource_type": "flavor", - "resoure_id": flavor["flavor-id"], - "resource-version": flavor["resource-version"] - }) + def countv0(self): + return len(self.queuev0) - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + def addv1(self, cloud_owner, cloud_region_id): + self.lock.acquire() + self.queuev1.append({"cloud-owner": cloud_owner, "cloud-region-id": cloud_region_id}) + self.lock.release() + return len(self.queuev1) - # remove all images - images = cloudregiondata.get("images", None) - for image in images.get("image", []) if images else []: - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" - "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource_type": "image", - "resoure_id": image["image-id"], - "resource-version": image["resource-version"] - }) - # remove image - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + def removev1(self, cloud_owner, cloud_region_id): + ''' + remove cloud region from list + ''' + self.queuev1 = [x for x in self.queuev1 if x["cloud-owner"] != cloud_owner or x["cloud-region-id"] != cloud_region_id] - # remove all az + def resetv1(self): + self.queuev1 = [] - # remove all vg + def countv1(self): + return len(self.queuev1) - # remove all snapshots - snapshots = cloudregiondata.get("snapshots", None) - for snapshot in snapshots.get("snapshot", []) if snapshots else []: - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s/" - "%(resource_type)ss/%(resource_type)s/%(resoure_id)s/" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource_type": "snapshot", - "resoure_id": snapshot["snapshot-id"], - "resource-version": snapshot["resource-version"] - }) - # remove snapshot - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + def state(self): + return self.state_ - # remove all server groups + def run(self): + logger.debug("Starting registration thread") + self.state_ = 1 + while self.helper and len(self.queuev0) > 0 and len(self.queuev1) > 0: + self.lock.acquire() + vimidv1 = self.queuev1.pop() + self.lock.release() + vimid = extsys.encode_vim_id(vimidv1["cloud-owner"], vimidv1["cloud-region-id"]) + self.helper(vimid) - # remove all pservers + self.lock.acquire() + vimidv0 = self.queuev0.pop() + self.lock.release() + self.helper(vimidv0) - # remove cloud region itself - resource_url = ("/cloud-infrastructure/cloud-regions/" - "cloud-region/%(cloud_owner)s/%(cloud_region_id)s" - "?resource-version=%(resource-version)s" - % { - "cloud_owner": cloud_owner, - "cloud_region_id": cloud_region_id, - "resource-version": cloudregiondata["resource-version"] - }) - # remove cloud region - retcode, content, status_code = \ - restcall.req_to_aai(resource_url, "DELETE") + self.state_ = 0 + # end of processing - #ret_code = VimDriverUtils.delete_vim_info(vimid) - return Response(status=status.HTTP_204_NO_CONTENT if retcode==0 else status.HTTP_500_INTERNAL_SERVER_ERROR) - except VimDriverNewtonException as e: - return Response(data={'error': e.content}, status=e.status_code) - except HttpError as e: - self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json())) - return Response(data=e.response.json(), status=e.http_status) - except Exception as e: - self._logger.error(traceback.format_exc()) - return Response(data={'error': str(e)}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR) diff --git a/share/starlingx_base/registration/registration.py b/share/starlingx_base/registration/registration.py index e07dda85..43a6b3e1 100644 --- a/share/starlingx_base/registration/registration.py +++ b/share/starlingx_base/registration/registration.py @@ -26,46 +26,156 @@ from common.msapi import extsys from keystoneauth1.exceptions import HttpError from newton_base.util import VimDriverUtils from common.utils import restcall +from threading import Thread logger = logging.getLogger(__name__) - - # DEBUG=True -class Registry(newton_registration.Registry): +# APIv0 handler upgrading: leverage APIv1 handler +class APIv0Registry(newton_registration.Registry): def __init__(self): - super(Registry, self).__init__() - self.proxy_prefix = settings.MULTICLOUD_PREFIX - self.aai_base_url = settings.AAI_BASE_URL + self.register_helper = RegistryHelper(settings.MULTICLOUD_PREFIX, settings.AAI_BASE_URL) + super(APIv0Registry, self).__init__() # self._logger = logger - def _get_ovsdpdk_capabilities(self, extra_specs, viminfo): - instruction_capability = {} - feature_uuid = uuid.uuid4() + def post(self, request, vimid=""): + self._logger.info("registration with : %s" % vimid) - instruction_capability['hpa-capability-id'] = str(feature_uuid) - instruction_capability['hpa-feature'] = 'ovsDpdk' - instruction_capability['architecture'] = 'Intel64' - instruction_capability['hpa-version'] = 'v1' + return super(APIv0Registry, self).post(request, vimid) - instruction_capability['hpa-feature-attributes'] = [] - instruction_capability['hpa-feature-attributes'].append( - {'hpa-attribute-key': 'dataProcessingAccelerationLibrary', - 'hpa-attribute-value': - '{{\"value\":\"{0}\"}}'.format("v17.02") - }) - return instruction_capability + def delete(self, request, vimid=""): + self._logger.debug("unregister cloud region: %s" % vimid) + return super(APIv0Registry, self).delete(request, vimid) + + +class Registry(APIv0Registry): + def __init__(self): + super(Registry, self).__init__() class APIv1Registry(newton_registration.Registry): def __init__(self): super(APIv1Registry, self).__init__() - self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX - self.aai_base_url = settings.AAI_BASE_URL + self.register_helper = RegistryHelper(settings.MULTICLOUD_API_V1_PREFIX, settings.AAI_BASE_URL) + # self._logger = logger + + def post(self, request, cloud_owner="", cloud_region_id=""): + self._logger.info("registration with : %s, %s" + % (cloud_owner, cloud_region_id)) + + try: + vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) + return super(APIv1Registry, self).post(request, vimid) + + except HttpError as e: + self._logger.error("HttpError: status:%s, response:%s" + % (e.http_status, e.response.json())) + return Response(data=e.response.json(), status=e.http_status) + except Exception as e: + self._logger.error(traceback.format_exc()) + return Response( + data={'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + def delete(self, request, cloud_owner="", cloud_region_id=""): + self._logger.debug("unregister cloud region: %s, %s" + % (cloud_owner, cloud_region_id)) + + vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) + return super(APIv1Registry, self).delete(request, vimid) + + +class RegistryHelper(newton_registration.RegistryHelper): + ''' + Helper code to discover and register a cloud region's resource + ''' + def __init__(self, multicloud_prefix, aai_base_url): + super(RegistryHelper, self).__init__(multicloud_prefix, aai_base_url) # self._logger = logger + def registry(self, vimid=""): + ''' + extend base method + ''' + viminfo = VimDriverUtils.get_vim_info(vimid) + cloud_extra_info_str = viminfo['cloud_extra_info'] + cloud_extra_info = None + try: + cloud_extra_info = json.loads(cloud_extra_info_str) \ + if cloud_extra_info_str else None + except Exception as ex: + logger.error("Can not convert cloud extra info %s %s" % ( + str(ex), cloud_extra_info_str)) + pass + + region_specified = cloud_extra_info.get( + "openstack-region-id", None) if cloud_extra_info else None + multi_region_discovery = cloud_extra_info.get( + "multi-region-discovery", None) if cloud_extra_info else None + + # set the default tenant since there is no tenant info in the VIM yet + sess = VimDriverUtils.get_session( + viminfo, tenant_name=viminfo['tenant']) + + # discover the regions, expect it always returns a list (even empty list) + # cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + # region_ids = self._discover_regions(cloud_owner, cloud_region_id, sess, viminfo) + region_ids = self._discover_regions(vimid, sess, viminfo) + + if len(region_ids) == 0: + self._logger.warn("failed to get region id") + + # compare the regions with region_specified and then cloud_region_id + if region_specified in region_ids: + pass + elif cloud_region_id in region_ids: + region_specified = cloud_region_id + pass + else: + # assume the first region be the primary region + # since we have no other way to determine it. + region_specified = region_ids.pop(0) + + # update cloud region and discover/register resource + if multi_region_discovery: + # no input for specified cloud region, + # so discover all cloud region + for regionid in region_ids: + # do not update the specified region here + if region_specified == regionid: + continue + + # create cloud region with composed AAI cloud_region_id + # except for the one onboarded externally (e.g. ESR) + gen_cloud_region_id = cloud_region_id + "_" + regionid + self._logger.info("create a cloud region: %s,%s,%s" + % (cloud_owner, gen_cloud_region_id, regionid)) + + self._update_cloud_region( + cloud_owner, gen_cloud_region_id, regionid, viminfo) + new_vimid = extsys.encode_vim_id( + cloud_owner, gen_cloud_region_id) + # super(APIv1Registry, self).post(request, new_vimid) + super(RegistryHelper, self)(new_vimid) + + # update the specified region + self._update_cloud_region(cloud_owner, cloud_region_id, + region_specified, viminfo) + + self.super(RegistryHelper, self).registry(vimid) + + return 0 + + + def unregistry(self, vimid=""): + '''extend base method''' + + return self.super(RegistryHelper, self).unregistry(vimid) + def _get_ovsdpdk_capabilities(self, extra_specs, viminfo): + '''extend base method''' + instruction_capability = {} feature_uuid = uuid.uuid4() @@ -174,11 +284,12 @@ class APIv1Registry(newton_registration.Registry): return retcode return 1 # unknown cloud owner,region_id - def _discover_regions(self, cloud_owner="", cloud_region_id="", + # def _discover_regions(self, cloud_owner="", cloud_region_id="", + def _discover_regions(self, vimid="", session=None, viminfo=None): try: regions = [] - vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) + # vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) isDistributedCloud = False openstackregions = self._get_list_resources( "/regions", "identity", session, viminfo, vimid, @@ -209,113 +320,3 @@ class APIv1Registry(newton_registration.Registry): except Exception: self._logger.error(traceback.format_exc()) return [] - - def post(self, request, cloud_owner="", cloud_region_id=""): - self._logger.info("registration with : %s, %s" - % (cloud_owner, cloud_region_id)) - - try: - vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) - - viminfo = VimDriverUtils.get_vim_info(vimid) - cloud_extra_info_str = viminfo['cloud_extra_info'] - cloud_extra_info = None - try: - cloud_extra_info = json.loads(cloud_extra_info_str)\ - if cloud_extra_info_str else None - except Exception as ex: - logger.error("Can not convert cloud extra info %s %s" % ( - str(ex), cloud_extra_info_str)) - pass - - region_specified = cloud_extra_info.get( - "openstack-region-id", None) if cloud_extra_info else None - multi_region_discovery = cloud_extra_info.get( - "multi-region-discovery", None) if cloud_extra_info else None - - # set the default tenant since there is no tenant info in the VIM yet - sess = VimDriverUtils.get_session( - viminfo, tenant_name=viminfo['tenant']) - - # discover the regions, expect it always returns a list (even empty list) - region_ids = self._discover_regions(cloud_owner, cloud_region_id, sess, viminfo) - - if len(region_ids) == 0: - self._logger.warn("failed to get region id") - - # compare the regions with region_specified and then cloud_region_id - if region_specified in region_ids: - pass - elif cloud_region_id in region_ids: - region_specified = cloud_region_id - pass - else: - # assume the first region be the primary region - # since we have no other way to determine it. - region_specified = region_ids.pop(0) - - # update cloud region and discover/register resource - if multi_region_discovery: - # no input for specified cloud region, - # so discover all cloud region - for regionid in region_ids: - # do not update the specified region here - if region_specified == regionid: - continue - - # create cloud region with composed AAI cloud_region_id - # except for the one onboarded externally (e.g. ESR) - gen_cloud_region_id = cloud_region_id + "_" + regionid - self._logger.info("create a cloud region: %s,%s,%s" - % (cloud_owner, gen_cloud_region_id, regionid)) - - self._update_cloud_region( - cloud_owner, gen_cloud_region_id, regionid, viminfo) - new_vimid = extsys.encode_vim_id( - cloud_owner, gen_cloud_region_id) - super(APIv1Registry, self).post(request, new_vimid) - - # update the specified region - self._update_cloud_region(cloud_owner, cloud_region_id, - region_specified, viminfo) - return super(APIv1Registry, self).post(request, vimid) - - except HttpError as e: - self._logger.error("HttpError: status:%s, response:%s" - % (e.http_status, e.response.json())) - return Response(data=e.response.json(), status=e.http_status) - except Exception as e: - self._logger.error(traceback.format_exc()) - return Response( - data={'error': str(e)}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR) - - def delete(self, request, cloud_owner="", cloud_region_id=""): - self._logger.debug("unregister cloud region: %s, %s" - % (cloud_owner, cloud_region_id)) - - vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) - return super(APIv1Registry, self).delete(request, vimid) - - -# APIv0 handler upgrading: leverage APIv1 handler -class APIv0Registry(APIv1Registry): - def __init__(self): - super(APIv0Registry, self).__init__() - self.proxy_prefix = settings.MULTICLOUD_PREFIX - self.aai_base_url = settings.AAI_BASE_URL - # self._logger = logger - - def post(self, request, vimid=""): - self._logger.info("registration with : %s" % vimid) - - cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) - return super(APIv0Registry, self).post( - request, cloud_owner, cloud_region_id) - - def delete(self, request, vimid=""): - self._logger.debug("unregister cloud region: %s" % vimid) - - cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) - return super(APIv0Registry, self).delete( - request, cloud_owner, cloud_region_id) diff --git a/starlingx/starlingx/registration/tests/test_registration2.py b/starlingx/starlingx/registration/tests/test_registration2.py index 62138540..6f470eea 100644 --- a/starlingx/starlingx/registration/tests/test_registration2.py +++ b/starlingx/starlingx/registration/tests/test_registration2.py @@ -86,8 +86,10 @@ class TestRegistration2(unittest.TestCase): ["get"], {"get": { "content": MOCK_GET_FLAVOR_RESPONSE}}), - resp = self.view._discover_flavors(vimid="starlingx_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="starlingx_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) @@ -101,7 +103,9 @@ class TestRegistration2(unittest.TestCase): "content": MOCK_GET_FLAVOR_EXTRASPECS_RESPONSE_w_hpa_numa} ]}), - resp = self.view._discover_flavors(vimid="starlingx_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors( + vimid="starlingx_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO + ) self.assertIsNone(resp) diff --git a/windriver/titanium_cloud/registration/tests/test_registration2.py b/windriver/titanium_cloud/registration/tests/test_registration2.py index 840f982f..495b2126 100644 --- a/windriver/titanium_cloud/registration/tests/test_registration2.py +++ b/windriver/titanium_cloud/registration/tests/test_registration2.py @@ -87,8 +87,8 @@ class TestRegistration2(unittest.TestCase): ["get"], {"get": { "content": MOCK_GET_FLAVOR_RESPONSE}}), - resp = self.view._discover_flavors(vimid="windriver-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors(vimid="windriver-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO) self.assertIsNone(resp) @@ -102,7 +102,7 @@ class TestRegistration2(unittest.TestCase): "content": MOCK_GET_FLAVOR_EXTRASPECS_RESPONSE_w_hpa_numa} ]}), - resp = self.view._discover_flavors(vimid="windriver-hudson-dc_RegionOne", - session=mock_session, viminfo=MOCK_VIM_INFO) + resp = self.view.register_helper._discover_flavors(vimid="windriver-hudson-dc_RegionOne", + session=mock_session, viminfo=MOCK_VIM_INFO) self.assertIsNone(resp) -- cgit 1.2.3-korg