diff options
Diffstat (limited to 'share/starlingx_base/registration/registration.py')
| -rw-r--r-- | share/starlingx_base/registration/registration.py | 226 |
1 files changed, 150 insertions, 76 deletions
diff --git a/share/starlingx_base/registration/registration.py b/share/starlingx_base/registration/registration.py index 255b6689..fa79e5b3 100644 --- a/share/starlingx_base/registration/registration.py +++ b/share/starlingx_base/registration/registration.py @@ -25,14 +25,18 @@ from newton_base.registration import registration as newton_registration from rest_framework import status from rest_framework.response import Response from common.msapi import extsys +from common.msapi import helper from keystoneauth1.exceptions import HttpError from newton_base.util import VimDriverUtils from common.utils import restcall -from threading import Thread +from django.core.cache import cache logger = logging.getLogger(__name__) -# DEBUG=True +# global var: Audition thread +gAZCapAuditThread = helper.MultiCloudThreadHelper() + +# DEBUG=True # APIv0 handler upgrading: leverage APIv1 handler class APIv0Registry(newton_registration.Registry): @@ -45,14 +49,24 @@ class APIv0Registry(newton_registration.Registry): self._logger.info("registration with : %s" % vimid) # vim registration will trigger the start the audit of AZ capacity - gAZCapAuditThread.addv0(vimid) + worker_self = InfraResourceAuditor( + settings.MULTICLOUD_API_V1_PREFIX, + settings.AAI_BASE_URL + ) + backlog_item = { + "id": vimid, + "worker": worker_self.azcap_audit, + "payload": (worker_self, vimid), + "repeat": 5*1000000, # repeat every 5 seconds + } + gAZCapAuditThread.add(backlog_item) if 0 == gAZCapAuditThread.state(): gAZCapAuditThread.start() return super(APIv0Registry, self).post(request, vimid) def delete(self, request, vimid=""): self._logger.debug("unregister cloud region: %s" % vimid) - gAZCapAuditThread.removev0(vimid) + gAZCapAuditThread.remove(vimid) return super(APIv0Registry, self).delete(request, vimid) @@ -63,8 +77,8 @@ class Registry(APIv0Registry): class APIv1Registry(newton_registration.Registry): def __init__(self): - super(APIv1Registry, self).__init__() self.register_helper = RegistryHelper(settings.MULTICLOUD_API_V1_PREFIX, settings.AAI_BASE_URL) + super(APIv1Registry, self).__init__() # self._logger = logger def post(self, request, cloud_owner="", cloud_region_id=""): @@ -75,7 +89,17 @@ class APIv1Registry(newton_registration.Registry): vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) # vim registration will trigger the start the audit of AZ capacity - gAZCapAuditThread.addv0(vimid) + worker_self = InfraResourceAuditor( + settings.MULTICLOUD_API_V1_PREFIX, + settings.AAI_BASE_URL + ) + backlog_item = { + "id": vimid, + "worker": worker_self.azcap_audit, + "payload": (worker_self, vimid), + "repeat": 5 * 1000000, # repeat every 5 seconds + } + gAZCapAuditThread.add(backlog_item) if 0 == gAZCapAuditThread.state(): gAZCapAuditThread.start() @@ -96,7 +120,7 @@ class APIv1Registry(newton_registration.Registry): % (cloud_owner, cloud_region_id)) vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) - gAZCapAuditThread.removev0(vimid) + gAZCapAuditThread.remove(vimid) return super(APIv1Registry, self).delete(request, vimid) @@ -108,7 +132,7 @@ class RegistryHelper(newton_registration.RegistryHelper): super(RegistryHelper, self).__init__(multicloud_prefix, aai_base_url) # self._logger = logger - def registry(self, vimid=""): + def registryV0(self, vimid=""): ''' extend base method ''' @@ -133,7 +157,7 @@ class RegistryHelper(newton_registration.RegistryHelper): viminfo, tenant_name=viminfo['tenant']) # discover the regions, expect it always returns a list (even empty list) - # cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) # region_ids = self._discover_regions(cloud_owner, cloud_region_id, sess, viminfo) region_ids = self._discover_regions(vimid, sess, viminfo) @@ -299,8 +323,7 @@ class RegistryHelper(newton_registration.RegistryHelper): return 1 # unknown cloud owner,region_id # def _discover_regions(self, cloud_owner="", cloud_region_id="", - def _discover_regions(self, vimid="", - session=None, viminfo=None): + def _discover_regions(self, vimid, session=None, viminfo=None): try: regions = [] # vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id) @@ -336,7 +359,7 @@ class RegistryHelper(newton_registration.RegistryHelper): return [] -class InfraResourceAuditor(object): +class InfraResourceAuditor(newton_registration.RegistryHelper): def __init__(self, multicloud_prefix, aai_base_url): self.proxy_prefix = multicloud_prefix @@ -344,70 +367,121 @@ class InfraResourceAuditor(object): self._logger = logger # super(InfraResourceAuditor, self).__init__(); - def azcap_audit(self, vimid=""): - # now retrieve the latest AZ cap info - # TBD - - # store the cap info into cache - # TBD - pass - - -class AuditorHelperThread(threading.Thread): - ''' - thread to register infrastructure resource into AAI - ''' - - def __init__(self, audit_helper): - threading.Thread.__init__(self) - self.daemon = True - self.duration = 0 - self.helper = audit_helper - - # The set of IDs of cloud regions, format: - # v0: "owner1_regionid1" - self.queuev0 = set() - self.lock = threading.Lock() - self.state_ = 0 # 0: stopped, 1: started - - def addv0(self, vimid=""): - self.lock.acquire() - self.queuev0.add(vimid) - self.lock.release() - return len(self.queuev0) - - def removev0(self, vimid): - ''' - discard cloud region from list without raise any exception - ''' - self.queuev0.discard(vimid) - - def resetv0(self): - self.queuev0.clear() - - def countv0(self): - return len(self.queuev0) - - def state(self): - return self.state_ - - def run(self): - logger.debug("Start the Audition thread") - self.state_ = 1 - while self.helper and self.countv0() > 0: - for vimidv0 in self.queuev0: - self.helper(vimidv0) - # sleep for a while in seconds - time.sleep(5) + def azcap_audit(self, vimid): + viminfo = VimDriverUtils.get_vim_info(vimid) + if not viminfo: + self._logger.warn("azcap_audit no valid vimid: %s" % vimid) + return - self.state_ = 0 - logger.debug("Stop the Audition thread") - # end of processing + session = VimDriverUtils.get_session( + viminfo, + tenant_name=viminfo['tenant'] + ) -# global Audition thread -gAZCapAuditThread = AuditorHelperThread( - InfraResourceAuditor( - settings.MULTICLOUD_API_V1_PREFIX, - settings.AAI_BASE_URL).azcap_audit -) + # now retrieve the latest AZ cap info + try: + # get all hypervisor detail ? + hypervisors = self._get_list_resources( + "/os-hypervisors/detail", "compute", session, + viminfo, vimid, "hypervisors") + + hypervisors_dict = {} + # for h in hypervisors: + # if not h.get("service", None): + # continue + # if not h.get("host", None): + # continue + # hypervisors_dict[h["service"]["host"]] = h + for h in hypervisors: + if not h.get("hypervisor_hostname", None): + continue + hypervisors_dict[h["hypervisor_hostname"]] = h + + vimAzCacheKey = "cap_azlist_" + vimid + vimAzList = [] + # cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + for az in self._get_list_resources( + "/os-availability-zone/detail", "compute", session, + viminfo, vimid, + "availabilityZoneInfo"): + az_info = { + 'availability-zone-name': az['zoneName'], + 'operational-status': az['zoneState']['available'] + if az.get('zoneState') else '', + 'hypervisor-type': '', + } + # filter out the default az: "internal" and "nova" + azName = az.get('zoneName', None) + # comment it for test the registration process only + # if azName == 'nova': + # continue + if azName == 'internal': + continue + vimAzList.append(azName) + + # get list of host names + pservers_info = [k for (k, v) in az['hosts'].items()] + + # Get current cap info of azName + azCapCacheKey = "cap_" + vimid + "_" + azName + azCapInfoCacheStr = cache.get(azCapCacheKey) + azCapInfoCache = json.loads(azCapInfoCacheStr) if azCapInfoCacheStr else None + + for psname in pservers_info: + psinfo = hypervisors_dict.get(psname, None) + if not psinfo: + # warning: the pserver info not found + continue + # get current pserver cap info + psCapInfoCacheKey = "cap_" + vimid + "_" + psname + psCapInfoCacheStr = cache.get(psCapInfoCacheKey) + psCapInfoCache = json.loads(psCapInfoCacheStr) if psCapInfoCacheStr else None + + # compare latest info with cached one + vcpu_delta = 0 + vcpu_used_delta = 0 + mem_delta = 0 + mem_free_delta = 0 + localstorage_delta = 0 + localstorage_free_delta = 0 + if psinfo.get("vcpus", 0) != psCapInfoCache.get("vcpus", 0): + vcpu_delta += psinfo.get("vcpus", 0) \ + - psCapInfoCache.get("vcpus", 0) + psCapInfoCache["vcpus"] = psinfo.get("vcpus", 0) + if psinfo.get("memory_mb", 0) != psCapInfoCache.get("memory_mb", 0): + mem_delta += psinfo.get("memory_mb", 0) \ + - psCapInfoCache.get("memory_mb", 0) + psCapInfoCache["memory_mb"] = psinfo.get("memory_mb", 0) + if psinfo.get("local_gb", 0) != psCapInfoCache.get("local_gb", 0): + localstorage_delta += psinfo.get("local_gb", 0) \ + - psCapInfoCache.get("local_gb", 0) + psCapInfoCache["local_gb"] = psinfo.get("local_gb", 0) + if psinfo.get("vcpus_used", 0) != psCapInfoCache.get("vcpus_used", 0): + vcpu_used_delta += psinfo.get("vcpus_used", 0)\ + - psCapInfoCache.get("vcpus_used", 0) + psCapInfoCache["vcpus_used"] = psinfo.get("vcpus_used", 0) + if psinfo.get("free_ram_mb", 0) != psCapInfoCache.get("free_ram_mb", 0): + mem_free_delta += psinfo.get("free_ram_mb", 0)\ + - psCapInfoCache.get("free_ram_mb", 0) + psCapInfoCache["free_ram_mb"] = psinfo.get("free_ram_mb", 0) + if psinfo.get("free_disk_gb", 0) != psCapInfoCache.get("free_disk_gb", 0): + localstorage_free_delta += psinfo.get("free_disk_gb", 0)\ + - psCapInfoCache.get("free_disk_gb", 0) + psCapInfoCache["free_disk_gb"] = psinfo.get("free_disk_gb", 0) + pass + + # now apply the delta to azCapInfo + azCapInfoCache["vcpus"] = azCapInfoCache.get("vcpus", 0) + vcpu_delta + azCapInfoCache["memory_mb"] = azCapInfoCache.get("memory_mb", 0) + mem_delta + azCapInfoCache["local_gb"] = azCapInfoCache.get("local_gb", 0) + localstorage_delta + azCapInfoCache["vcpus_used"] = azCapInfoCache.get("vcpus_used", 0) + vcpu_used_delta + azCapInfoCache["free_ram_mb"] = azCapInfoCache.get("free_ram_mb", 0) + mem_free_delta + azCapInfoCache["free_disk_gb"] = azCapInfoCache.get("free_disk_gb", 0) + localstorage_free_delta + + # update the cache + cache.set(azCapCacheKey, json.dumps(azCapInfoCache), 3600 * 24) + cache.set(vimAzCacheKey, vimAzList, 3600 * 24) + except Exception as e: + self._logger.error("azcap_audit raise exception: %s" % e) + pass |