summaryrefslogtreecommitdiffstats
path: root/windriver/titanium_cloud/vesagent/vesagent_ctrl.py
diff options
context:
space:
mode:
Diffstat (limited to 'windriver/titanium_cloud/vesagent/vesagent_ctrl.py')
-rw-r--r--windriver/titanium_cloud/vesagent/vesagent_ctrl.py452
1 files changed, 0 insertions, 452 deletions
diff --git a/windriver/titanium_cloud/vesagent/vesagent_ctrl.py b/windriver/titanium_cloud/vesagent/vesagent_ctrl.py
deleted file mode 100644
index 86b4ef51..00000000
--- a/windriver/titanium_cloud/vesagent/vesagent_ctrl.py
+++ /dev/null
@@ -1,452 +0,0 @@
-# Copyright (c) 2017-2018 Wind River Systems, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-# import traceback
-import json
-
-from rest_framework import status
-from rest_framework.response import Response
-from rest_framework.views import APIView
-
-from django.conf import settings
-from common.msapi import extsys
-from titanium_cloud.vesagent.tasks import scheduleBacklogs
-from titanium_cloud.vesagent.event_domain import fault_vm
-
-from django.core.cache import cache
-
-logger = logging.getLogger(__name__)
-
-
-class VesAgentCtrl(APIView):
- '''
- control plane of VesAgent
- Design tips:
- 1, vesagent are multiple processing workers
- 2, the runtime logic is simple: a vesagent worker polls the data source (vm/hypervisor/host/vim/etc.)
- and then feeds the encoded data to VES.
- 3, the vesagent workers can be distributed to different clouds while latency/throughput is concerned,
- this distributed deployment usually comes along with the distributed VES deployment.
- So it is very likely that the collected data from different VIM/Cloud instance will be fed into
- different VES endpoint, however, assuming that there will be at most one VES endpoint serving
- any single VIM/Cloud instance.
- 4, According to VES specs, the collected data can be cataloged by domain:
- domain : fault, heartbeat, measurementsForVfScaling, other, stateChange, syslog, thresholdCrossingAlert
- As far as VIM/Cloud concerned, fault, heartbeat, measurementsForVfScaling, TCAalert are relevant.
- 5, the source of the collected data can be cataloged by eventSourceType:
- eventSourceType: VNF/VNFC/VM
- As far as VIM/Cloud concerned, only VM is relevant. This eventSourceType should be extended to cover
- the data source of hypervisor, VIM, Host,Controller, PIM, etc.
-
- 6, the source of collected data should be specified explicitly,so is the domain of the collected data.
- To specify the source: eventSourceType, uuid or name of the source
- To specify the domain: domain
- the specifications above will be provisioned as a vesagent backlog entry to a VIM/Cloud instance
- to tell a vesagent worker that :
- with regarding to that VIM/Cloud instance, what kind of data to be collected from which source .
-
- 7,the VES endpoint will be also specified for a VIM/Cloud instance, so that all collected data
- will be fed into this VES endpoint
-
- 8, the vesagent backlog are stored into the respective cloud_region's property "cloud-extra-info",
- which implies that those specifications can be CRUD either by ESR portal or the RestAPIs in this view, e.g.
- "cloud-extra-info": {
- ...,
- "vesagent_config":
- {
- "ves_subscription":{
- "endpoint":"http://{VES IP}:{VES port}/{URI}",
- "username":"{VES username}",
- "password":"{VES password}",
- },
- "poll_interval_default" : "{default interval for polling}",
- "backlogs":[
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name1}",
- "source":"{VM name1}",
- "poll_interval" : "{optional, interval for polling}",
- },
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name2}",
- "source":"{VM name2}",
- "poll_interval" : "{optional, interval for polling}",
- }
- ]
- }
- }
-
- Idea: API dispatching to distributed M.C. service can be determined by Complex Object in AAI:
- cloud-region has been assoicated to a Complex Object
- M.C. plugin service instance should refer to the same Complex Object (by physical_locaton_id ?)
- So the M.C. broker/API distributor/other approach will correlate the cloud-region with
- corresponding M.C. plugin service instance.
-
-
- Backlog built in cache:
-
- maintain backlog in cache and VES agent workers
- cache objects:
- "VesAgentBacklogs.vimlist": [ list of vimid] ### will not expire forever
- "VesAgentBacklogs.state.{vimdid}":
- ### will expire eventually to eliminate the garbage, expiration duration: 1hour?
- {
- "{backlog_uuid}": {
- "timestamp": "{timestamp for last time of data collecting}",
- "api_data": [list of data to populate the format string of the API link]
- "last_event": {object, event reported to ves last time}"
- }
- }
- "VesAgentBacklogs.config.{vimdid}": ### will not expire forever
- {
- "vimid": "{vim id}",
- "subscription": {
- "endpoint": "{ves endpoint, e.g. http://ves_ip:ves_port/eventListener/v5}",
- "username": "{username}",
- "password": "{password}"
- }
- "poll_interval_default" : "{default interval for polling}",
- "backlogs":[
- {
- "backlog_uuid": "{uuid to identify the backlog}"
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name1}",
- "source":"{VM name1}",
- "poll_interval" : "{optional, interval in second for polling}",
- "api_method": "{GET/POST/PUT/etc.}",
- "api_link":"{API link to collect data, could be format string}",
- "tenant_id": tenant_id,
- "server_id": server_id,
- },
- {
- "domain":"fault"
- "type":"vm",
- "tenant":"{tenant name2}",
- "source":"{VM name2}",
- "poll_interval" : "{optional, interval in second for polling}",
- "api_method": "{GET/POST/PUT/etc.}",
- "api_link":"{API link to collect data, could be format string}",
- "tenant_id": tenant_id,
- "server_id": server_id,
- }
- ]
- }
- '''
-
- def __init__(self):
- self._logger = logger
- self.proxy_prefix = settings.MULTICLOUD_PREFIX
-
- def get(self, request, vimid=""):
- '''
- get blob of vesagent-config
- :param request:
- :param vimid:
- :return:
- '''
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s" % request.META)
- try:
- # get vesagent_config from cloud region
- try:
- viminfo = extsys.get_vim_by_id(vimid)
- cloud_extra_info_str = viminfo.get('cloud_extra_info', '')
- cloud_extra_info = json.loads(cloud_extra_info_str) if cloud_extra_info_str != '' else None
- vesagent_config = cloud_extra_info.get("vesagent_config", None) if cloud_extra_info is not None else None
- except Exception:
- # ignore this error
- self._logger.warn("cloud extra info is provided with data in bad format: %s" % cloud_extra_info_str)
- pass
-
- vesagent_backlogs = self.getBacklogsOneVIM(vimid)
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_200_OK)
- return Response(data={"vesagent_config": vesagent_config,
- "vesagent_backlogs": vesagent_backlogs},
- status=status.HTTP_200_OK)
-
- def post(self, request, vimid=""):
- '''
- update the blob of vesagent-config, rebuild the backlog for the vesagent workers,
- and start the vesagent workers if not started yet
- Implication: the request to this API endpoint will build the backlog locally, hence only local VES agent workers
- will process these backlogs, which conforms to distributed deployment of M.C. services which includes VES agents
- :param request:{"vesagent_config":
- {"ves_subscription":
- {"endpoint":"http://127.0.0.1:9005/sample",
- "username":"user","password":"password"},
- "poll_interval_default":10,
- "backlogs":
- [
- {"domain":"fault","type":"vm","tenant":"VIM","source":"onap-aaf"}
- ]
- }
- }
- :param vimid:
- :return:
- '''
-
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s, with data: %s" % (request.META, request.data))
- try:
- vesagent_config = None
- if request.data is None or request.data.get("vesagent_config", None) is None:
- # Try to load the vesagent_config out of cloud_region["cloud_extra_info"]
- viminfo = extsys.get_vim_by_id(vimid)
- cloud_extra_info_str = viminfo.get('cloud_extra_info', None)
- cloud_extra_info = json.loads(cloud_extra_info_str) if cloud_extra_info_str is not None else None
- vesagent_config = cloud_extra_info.get("vesagent_config", None) if cloud_extra_info is not None else None
- else:
- vesagent_config = request.data.get("vesagent_config", None)
-
- if vesagent_config is None:
- return Response(data={'vesagent_config is not provided'},
- status=status.HTTP_400_BAD_REQUEST)
-
- vesagent_backlogs = self.buildBacklogsOneVIM(vimid, vesagent_config)
-
- # store back to cloud_extra_info
- # tbd
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_201_CREATED)
- return Response(data={"vesagent_config": vesagent_config,
- "vesagent_backlogs": vesagent_backlogs},
- status=status.HTTP_201_CREATED)
-
- def delete(self, request, vimid=""):
- '''
- delete the blob of vesagent-config, remove it from backlog and stop the vesagent worker if no backlog
- :param request:
- :param vimid:
- :return:
- '''
- self._logger.info("vimid: %s" % vimid)
- self._logger.debug("with META: %s" % request.META)
- try:
- # tbd
- self.clearBacklogsOneVIM(vimid)
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return Response(data={'error': str(e)},
- status=status.HTTP_500_INTERNAL_SERVER_ERROR)
-
- self._logger.info("return with %s" % status.HTTP_200_OK)
- return Response(status=status.HTTP_200_OK)
-
- def getBacklogsOneVIM(self, vimid):
- '''
- remove the specified backlogs for a VIM
- :param vimid:
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
-
- vesAgentConfig = None
- try:
- # retrive the backlogs
- vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid))
- if vesAgentConfigStr is None:
- logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid))
- return None
-
- logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr))
-
- vesAgentConfig = json.loads(vesAgentConfigStr)
- if vesAgentConfig is None:
- logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid))
- return None
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- vesAgentConfig = {"error": "exception occurs"}
-
- self._logger.debug("return")
- return vesAgentConfig
-
- def clearBacklogsOneVIM(self, vimid):
- '''
- remove the specified backlogs for a VIM
- :param vimid:
- :param vesagent_config:
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
-
- try:
- # remove vimid from "VesAgentBacklogs.vimlist"
- VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist")
- VesAgentBacklogsVimList = []
- if VesAgentBacklogsVimListStr is not None:
- VesAgentBacklogsVimList = json.loads(VesAgentBacklogsVimListStr)
- VesAgentBacklogsVimList = [v for v in VesAgentBacklogsVimList if v != vimid]
-
- logger.debug("VesAgentBacklogs.vimlist is %s" % VesAgentBacklogsVimList)
-
- # cache forever
- cache.set("VesAgentBacklogs.vimlist", json.dumps(VesAgentBacklogsVimList), None)
-
- # retrieve the backlogs
- vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid))
- if vesAgentConfigStr is None:
- logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid))
- return 0
-
- logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr))
-
- vesAgentConfig = json.loads(vesAgentConfigStr)
- if vesAgentConfig is None:
- logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid))
- return 0
-
- # iterate all backlog and remove the associate state!
- # tbd
-
- # clear the whole backlogs for a VIM
- cache.set("VesAgentBacklogs.config.%s" % vimid, "deleting the backlogs", 1)
-
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
-
- self._logger.debug("return")
- return 0
-
- def buildBacklogsOneVIM(self, vimid, vesagent_config=None):
- '''
- build and cache backlog for specific cloud region,spawn vesagent workers if needed
- :param vimid:
- :param vesagent_config: vesagent_config data in json object
- :return:
- '''
- self._logger.debug("vimid: %s" % vimid)
- self._logger.debug("config data: %s" % vesagent_config)
-
- VesAgentBacklogsConfig = None
- try:
- if vesagent_config:
- # now rebuild the backlog
- VesAgentBacklogsConfig = {
- "vimid": vimid,
- "poll_interval_default": vesagent_config.get("poll_interval_default", 0),
- "subscription": vesagent_config.get("ves_subscription", None),
- "backlogs": [self.buildBacklog(vimid, b) for b in vesagent_config.get("backlogs", [])]
- }
-
- # add/update the backlog into cache
- VesAgentBacklogsConfigStr = json.dumps(VesAgentBacklogsConfig)
- # cache forever
- cache.set("VesAgentBacklogs.config.%s" % vimid, VesAgentBacklogsConfigStr, None)
-
- # update list of vimid for vesagent
- # get the whole list of backlog
- VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist")
- VesAgentBacklogsVimList = [vimid]
- if VesAgentBacklogsVimListStr is not None:
- VesAgentBacklogsVimList = json.loads(VesAgentBacklogsVimListStr)
- VesAgentBacklogsVimList = [v for v in VesAgentBacklogsVimList if v != vimid]
- VesAgentBacklogsVimList.append(vimid)
-
- logger.debug("VesAgentBacklogs.vimlist is %s" % VesAgentBacklogsVimList)
-
- # cache forever
- cache.set("VesAgentBacklogs.vimlist", json.dumps(VesAgentBacklogsVimList), None)
-
- # notify schduler
- scheduleBacklogs.delay(vimid)
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- VesAgentBacklogsConfig = {"error": "exception occurs during build backlogs"}
-
- self._logger.debug("return")
- return VesAgentBacklogsConfig
-
- def buildBacklog(self, vimid, backlog_input):
- self._logger.debug("build backlog for: %s" % vimid)
- self._logger.debug("with input: %s" % backlog_input)
-
- try:
- if backlog_input["domain"] == "fault" and backlog_input["type"] == "vm":
- return fault_vm.buildBacklog_fault_vm(vimid, backlog_input)
- else:
- self._logger.warn("return with failure: unsupported backlog domain:%s, type:%s"
- % (backlog_input["domain"], backlog_input["type"] == "vm"))
- return None
- except Exception as e:
- self._logger.error("exception:%s" % str(e))
- return None
-
- self._logger.debug("return without backlog")
- return None
-
-
-class APIv1VesAgentCtrl(VesAgentCtrl):
-
- def __init__(self):
- super(APIv1VesAgentCtrl, self).__init__()
- # self._logger = logger
- self.proxy_prefix = settings.MULTICLOUD_API_V1_PREFIX
-
- def get(self, request, cloud_owner="", cloud_region_id=""):
- '''
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info("cloud_owner, cloud_region_id: %s,%s" % (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).get(request, vimid)
-
- def post(self, request, cloud_owner="", cloud_region_id=""):
- '''
- wrapper for inherited API with VIM ID
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info("cloud_owner, cloud_region_id: %s,%s" % (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).post(request, vimid)
-
- def delete(self, request, cloud_owner="", cloud_region_id=""):
- '''
- wrapper of inherited API with VIM ID
- :param request:
- :param cloud_owner:
- :param cloud_region_id:
- :return:
- '''
- self._logger.info(
- "cloud_owner, cloud_region_id: %s,%s" %
- (cloud_owner, cloud_region_id))
-
- vimid = extsys.encode_vim_id(cloud_owner, cloud_region_id)
- return super(APIv1VesAgentCtrl, self).delete(request, vimid)