diff options
| -rw-r--r-- | windriver/titanium_cloud/vesagent/event_domain/fault_vm.py | 174 | ||||
| -rw-r--r-- | windriver/titanium_cloud/vesagent/tasks.py | 90 | ||||
| -rw-r--r-- | windriver/titanium_cloud/vesagent/tests.py | 60 | ||||
| -rw-r--r-- | windriver/titanium_cloud/vesagent/vesagent_ctrl.py | 100 | ||||
| -rw-r--r-- | windriver/titanium_cloud/vesagent/vespublish.py | 43 |
5 files changed, 465 insertions, 2 deletions
diff --git a/windriver/titanium_cloud/vesagent/event_domain/fault_vm.py b/windriver/titanium_cloud/vesagent/event_domain/fault_vm.py index 314847c0..308ef24c 100644 --- a/windriver/titanium_cloud/vesagent/event_domain/fault_vm.py +++ b/windriver/titanium_cloud/vesagent/event_domain/fault_vm.py @@ -17,8 +17,10 @@ import logging import json import uuid +import time from django.conf import settings +from titanium_cloud.vesagent.vespublish import publishAnyEventToVES from common.utils.restcall import _call_req logger = logging.getLogger(__name__) @@ -108,3 +110,175 @@ def buildBacklog_fault_vm(vimid, backlog_input): logger.debug("with backlog: %s" % backlog) return backlog + +### process backlog with domain:"fault", type:"vm" + + + +def processBacklog_fault_vm(vesAgentConfig, vesAgentState, oneBacklog): + logger.debug("vesAgentConfig:%s, vesAgentState:%s, oneBacklog: %s" + % (vesAgentConfig, vesAgentState, oneBacklog)) + + try: + vimid = vesAgentConfig["vimid"] + tenant_name = oneBacklog["tenant"] + + # get token + auth_api_url_format = "/{f_vim_id}/identity/v2.0/tokens" + auth_api_url = auth_api_url_format.format(f_vim_id=vimid) + auth_api_data = { "auth":{"tenantName": tenant_name} } + base_url = settings.MULTICLOUD_PREFIX + extra_headers = '' + logger.debug("authenticate with url:%s" % auth_api_url) + ret = _call_req(base_url, "", "", 0, auth_api_url, "POST", extra_headers, json.dumps(auth_api_data)) + if ret[0] > 0 or ret[1] is None: + logger.critical("call url %s failed with status %s" % (auth_api_url, ret[0])) + + token_resp = json.JSONDecoder().decode(ret[1]) + logger.debug("authenticate resp: %s" % token_resp) + token = token_resp["access"]["token"]["id"] + + # collect data by issue API + api_link = oneBacklog["api_link"] + method = oneBacklog["api_method"] + base_url = settings.MULTICLOUD_PREFIX + data = '' + extra_headers = {'X-Auth-Token': token} + #which one is correct? extra_headers = {'HTTP_X_AUTH_TOKEN': token} + logger.debug("authenticate with url:%s, header:%s" % (auth_api_url,extra_headers)) + ret = _call_req(base_url, "", "", 0, api_link, method, extra_headers, data) + if ret[0] > 0 or ret[1] is None: + logger.critical("call url %s failed with status %s" % (api_link, ret[0])) + + server_resp = json.JSONDecoder().decode(ret[1]) + logger.debug("collected data: %s" % server_resp) + + # encode data + backlog_uuid = oneBacklog.get("backlog_uuid", None) + backlogState = vesAgentState.get("%s" % (backlog_uuid), None) + last_event = backlogState.get("last_event", None) + logger.debug("last event: %s" % last_event) + + this_event = data2event_fault_vm(oneBacklog, last_event, server_resp) + + if this_event is not None: + logger.debug("this event: %s" % this_event) + # report data to VES + ves_subscription = vesAgentConfig.get("subscription", None) + publishAnyEventToVES(ves_subscription, this_event) + # store the latest data into cache, never expire + backlogState["last_event"] = this_event + + except Exception as e: + logger.error("exception:%s" % str(e)) + return + + logger.info("return") + return + + +def data2event_fault_vm(oneBacklog, last_event, vm_data): + + VES_EVENT_VERSION = 3.0 + VES_EVENT_FAULT_VERSION = 2.0 + VES_EVENT_FAULT_DOMAIN = "fault" + + try: + + if vm_status_is_fault(vm_data["server"]["status"]): + if last_event is not None \ + and last_event['event']['commonEventHeader']['eventName'] == 'Fault_MultiCloud_VMFailure': + # asserted alarm already, so no need to assert it again + return None + + eventName = "Fault_MultiCloud_VMFailure" + priority = "High" + eventSeverity = "CRITICAL" + alarmCondition = "Guest_Os_Failure" + vfStatus = "Active" + specificProblem = "AlarmOn" + eventType = '' + reportingEntityId = '' + reportingEntityName = '' + sequence = 0 + + startEpochMicrosec = int(time.time()) + lastEpochMicrosec = int(time.time()) + + eventId = str(uuid.uuid4()) + pass + else: + if last_event is None \ + or last_event['event']['commonEventHeader']['eventName'] != 'Fault_MultiCloud_VMFailure': + # not assert alarm yet, so no need to clear it + return None + + + eventName = "Fault_MultiCloud_VMFailureCleared" + priority = "Normal" + eventSeverity = "NORMAL" + alarmCondition = "Vm_Restart" + vfStatus = "Active" + specificProblem = "AlarmOff" + eventType = '' + reportingEntityId = '' + reportingEntityName = '' + sequence = 0 + + startEpochMicrosec = last_event['event']['commonEventHeader']['startEpochMicrosec'] + lastEpochMicrosec = int(time.time()) + eventId = last_event['event']['commonEventHeader']['eventId'] + + pass + + # now populate the event structure + this_event = { + 'event': { + 'commonEventHeader': { + 'version': VES_EVENT_VERSION, + 'eventName': eventName, + 'domain': VES_EVENT_FAULT_DOMAIN, + 'eventId': eventId, + 'eventType': eventType, + 'sourceId': vm_data["server"]['id'], + 'sourceName': vm_data["server"]['name'], + 'reportingEntityId': reportingEntityId, + 'reportingEntityName': reportingEntityName, + 'priority': priority, + 'startEpochMicrosec': startEpochMicrosec, + 'lastEpochMicrosec': lastEpochMicrosec, + 'sequence': sequence + }, + 'faultFields': { + 'faultFieldsVersion': VES_EVENT_FAULT_VERSION, + 'eventSeverity': eventSeverity, + 'eventSourceType': 'virtualMachine', + 'alarmCondition': alarmCondition, + 'specificProblem': specificProblem, + 'vfStatus': 'Active' + } + + } + + } + + return this_event + + except Exception as e: + logger.error("exception:%s" % str(e)) + return None + + +def vm_status_is_fault(status): + ''' + report VM fault when status falls into one of following state + ['ERROR', 'DELETED', 'PAUSED', 'REBUILD', 'RESCUE', + 'RESIZE','REVERT_RESIZE', 'SHELVED', 'SHELVED_OFFLOADED', + 'SHUTOFF', 'SOFT_DELETED','SUSPENDED', 'UNKNOWN', 'VERIFY_RESIZE'] + :param status: + :return: + ''' + if status in ['BUILD', 'ACTIVE', 'HARD_REBOOT', 'REBOOT', 'MIGRATING', 'PASSWORD']: + return False + else: + return True diff --git a/windriver/titanium_cloud/vesagent/tasks.py b/windriver/titanium_cloud/vesagent/tasks.py index e46ed8a7..35ccfcaf 100644 --- a/windriver/titanium_cloud/vesagent/tasks.py +++ b/windriver/titanium_cloud/vesagent/tasks.py @@ -22,6 +22,8 @@ import time from django.core.cache import cache +from titanium_cloud.vesagent.event_domain.fault_vm import processBacklog_fault_vm + logger = logging.getLogger(__name__) @@ -83,7 +85,7 @@ def processBacklogsOfOneVIM(vimid): :param vimid: :return: ''' - backlog_count = 3 + backlog_count = 0 next_time_slot = 10 try: @@ -100,9 +102,95 @@ def processBacklogsOfOneVIM(vimid): return 0,next_time_slot + vesAgentStateStr = cache.get("VesAgentBacklogs.state.%s" % (vimid)) + vesAgentState = json.loads(vesAgentStateStr) if vesAgentStateStr is not None else {} + + ves_info = vesAgentConfig.get("subscription", None) + if ves_info is None: + logger.warn("VesAgentBacklogs.config.%s: ves subscription corrupts:%s" % (vimid, vesAgentConfigStr)) + return 0,next_time_slot + + poll_interval_default = vesAgentConfig.get("poll_interval_default", None) + if poll_interval_default is None: + logger.warn("VesAgentBacklogs.config.%s: poll_interval_default corrupts:%s" % (vimid, vesAgentConfigStr)) + return 0,next_time_slot + + if poll_interval_default == 0: + # invalid interval value + logger.warn("VesAgentBacklogs.config.%s: poll_interval_default invalid:%s" % (vimid, vesAgentConfigStr)) + return 0,next_time_slot + + backlogs_list = vesAgentConfig.get("backlogs", None) + if backlogs_list is None: + logger.warn("VesAgentBacklogs.config.%s: backlogs corrupts:%s" % (vimid, vesAgentConfigStr)) + return 0,next_time_slot + + for backlog in backlogs_list: + backlog_count_tmp, next_time_slot_tmp = processOneBacklog( + vesAgentConfig, vesAgentState, poll_interval_default, backlog) + logger.debug("processOneBacklog return with %s,%s" % (backlog_count_tmp, next_time_slot_tmp)) + backlog_count += backlog_count_tmp + next_time_slot = next_time_slot_tmp if next_time_slot > next_time_slot_tmp else next_time_slot + + pass + + # save back the updated backlogs state + vesAgentStateStr = json.dumps(vesAgentState) + cache.set("VesAgentBacklogs.state.%s" % vimid, vesAgentStateStr, None) + + except Exception as e: + logger.error("exception:%s" % str(e)) + + return backlog_count, next_time_slot + + +def processOneBacklog(vesAgentConfig, vesAgentState, poll_interval_default, oneBacklog): + logger.info("Process one backlog") + #logger.debug("vesAgentConfig:%s, vesAgentState:%s, poll_interval_default:%s, oneBacklog: %s" + # % (vesAgentConfig, vesAgentState, poll_interval_default, oneBacklog)) + + backlog_count = 1 + next_time_slot = 10 + try: + timestamp_now = int(time.time()) + backlog_uuid = oneBacklog.get("backlog_uuid", None) + if backlog_uuid is None: + # warning: uuid is None, omit this backlog + logger.warn("backlog without uuid: %s" % oneBacklog) + return 0, next_time_slot + + backlogState = vesAgentState.get("%s" % (backlog_uuid), None) + if backlogState is None: + initialBacklogState = { + "timestamp": timestamp_now + } + vesAgentState["%s" % (backlog_uuid)] = initialBacklogState + backlogState = initialBacklogState + + time_expiration = backlogState["timestamp"] \ + + oneBacklog.get("poll_interval", poll_interval_default) + # check if poll interval expires + if timestamp_now < time_expiration: + # not expired yet + logger.info("return without dispatching, not expired yet") + return backlog_count, next_time_slot + + logger.info("Dispatching backlog") + + # collect data in case of expiration + if oneBacklog["domain"] == "fault" and oneBacklog["type"] == "vm": + processBacklog_fault_vm(vesAgentConfig, vesAgentState, oneBacklog) + else: + logger.warn("Dispatching backlog fails due to unsupported backlog domain %s,type:%s" + % (oneBacklog["domain"], oneBacklog["type"])) + backlog_count = 0 + pass + # update timestamp and internal state + backlogState["timestamp"] = timestamp_now except Exception as e: logger.error("exception:%s" % str(e)) + logger.info("return") return backlog_count, next_time_slot diff --git a/windriver/titanium_cloud/vesagent/tests.py b/windriver/titanium_cloud/vesagent/tests.py new file mode 100644 index 00000000..7026d569 --- /dev/null +++ b/windriver/titanium_cloud/vesagent/tests.py @@ -0,0 +1,60 @@ +# Copyright (c) 2017-2018 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import unittest +import json +from django.test import Client +from rest_framework import status + +from django.core.cache import cache +from common.msapi import extsys + + + +MOCK_VIM_INFO = { + "createTime": "2017-04-01 02:22:27", + "domain": "Default", + "name": "TiS_R4", + "password": "admin", + "tenant": "admin", + "type": "openstack", + "url": "http://128.224.180.14:5000/v3", + "userName": "admin", + "vendor": "WindRiver", + "version": "newton", + "vimId": "windriver-hudson-dc_RegionOne", + 'cloud_owner': 'windriver-hudson-dc', + 'cloud_region_id': 'RegionOne', + 'cloud_extra_info': '{"vesagent_config":{"backlogs":[{"source":"onap-aaf","domain":"fault","type":"vm","tenant":"VIM"}],"poll_interval_default":10,"ves_subscription":{"username":"user","password":"password","endpoint":"http://127.0.0.1:9005/sample"}}}', + 'cloud_epa_caps': '', + 'insecure': 'True', +} + +class VesAgentCtrlTest(unittest.TestCase): + def setUp(self): + self.client = Client() + + def tearDown(self): + pass + + @mock.patch.object(cache, 'get') + @mock.patch.object(extsys, 'get_vim_by_id') + def test_get(self, mock_get_vim_by_id, mock_get): + mock_get_vim_by_id.return_value = MOCK_VIM_INFO + mock_get.return_value = '{"backlogs": [{"backlog_uuid": "2b8f6ff8-bc64-339b-a714-155909db937f", "server_id": "c4b575fa-ed85-4642-ab4b-335cb5744721", "tenant_id": "0e148b76ee8c42f78d37013bf6b7b1ae", "api_method": "GET", "source": "onap-aaf", "api_link": "/onaplab_RegionOne/compute/v2.1/0e148b76ee8c42f78d37013bf6b7b1ae/servers/c4b575fa-ed85-4642-ab4b-335cb5744721", "domain": "fault", "type": "vm", "tenant": "VIM"}], "poll_interval_default": 10, "vimid": "onaplab_RegionOne", "subscription": {"username": "user", "password": "password", "endpoint": "http://127.0.0.1:9005/sample"}}' + + response = self.client.get("/api/multicloud-titanium_cloud/v0/windriver-hudson-dc_RegionOne/vesagent") + self.assertEqual(status.HTTP_200_OK, response.status_code, response.content) diff --git a/windriver/titanium_cloud/vesagent/vesagent_ctrl.py b/windriver/titanium_cloud/vesagent/vesagent_ctrl.py index a531a61a..fdc9f71a 100644 --- a/windriver/titanium_cloud/vesagent/vesagent_ctrl.py +++ b/windriver/titanium_cloud/vesagent/vesagent_ctrl.py @@ -162,8 +162,23 @@ class VesAgentCtrl(APIView): ''' self._logger.info("vimid: %s" % vimid) self._logger.debug("with META: %s" % request.META) + try: + # get vesagent_config from cloud region + viminfo = extsys.get_vim_by_id(vimid) + cloud_extra_info_str = viminfo.get('cloud_extra_info', None) + cloud_extra_info = json.loads(cloud_extra_info_str) if cloud_extra_info_str is not None else None + vesagent_config = cloud_extra_info.get("vesagent_config", None) if cloud_extra_info is not None else None - pass + vesagent_backlogs = self.getBacklogsOneVIM(vimid) + + except Exception as e: + self._logger.error("exception:%s" % str(e)) + return Response(data={'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + return Response(data={"vesagent_config":vesagent_config, + "vesagent_backlogs": vesagent_backlogs}, + status=status.HTTP_200_OK) def post(self, request, vimid=""): @@ -226,10 +241,93 @@ class VesAgentCtrl(APIView): ''' self._logger.info("vimid: %s" % vimid) self._logger.debug("with META: %s" % request.META) + try: + # tbd + self.clearBacklogsOneVIM(vimid) + except Exception as e: + self._logger.error("exception:%s" % str(e)) + return Response(data={'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) return Response(status=status.HTTP_200_OK) + def getBacklogsOneVIM(self, vimid): + ''' + remove the specified backlogs for a VIM + :param vimid: + :return: + ''' + self._logger.info("vimid: %s" % vimid) + + vesAgentConfig = None + try: + # retrive the backlogs + vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid)) + if vesAgentConfigStr is None: + logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid)) + return None + + logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr)) + + vesAgentConfig = json.loads(vesAgentConfigStr) + if vesAgentConfig is None: + logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid)) + return None + + except Exception as e: + self._logger.error("exception:%s" % str(e)) + vesAgentConfig = {"error": "exception occurs"} + + self._logger.info("return") + return vesAgentConfig + + def clearBacklogsOneVIM(self, vimid): + ''' + remove the specified backlogs for a VIM + :param vimid: + :param vesagent_config: + :return: + ''' + self._logger.info("vimid: %s" % vimid) + + try: + # remove vimid from "VesAgentBacklogs.vimlist" + VesAgentBacklogsVimListStr = cache.get("VesAgentBacklogs.vimlist") + VesAgentBacklogsVimList = [] + if VesAgentBacklogsVimListStr is not None: + VesAgentBacklogsVimList = json.loads(VesAgentBacklogsVimListStr) + VesAgentBacklogsVimList = [v for v in VesAgentBacklogsVimList if v != vimid] + + logger.info("VesAgentBacklogs.vimlist is %s" % VesAgentBacklogsVimList) + + # cache forever + cache.set("VesAgentBacklogs.vimlist", json.dumps(VesAgentBacklogsVimList), None) + + # retrieve the backlogs + vesAgentConfigStr = cache.get("VesAgentBacklogs.config.%s" % (vimid)) + if vesAgentConfigStr is None: + logger.warn("VesAgentBacklogs.config.%s cannot be found in cache" % (vimid)) + return 0 + + logger.debug("VesAgentBacklogs.config.%s: %s" % (vimid, vesAgentConfigStr)) + + vesAgentConfig = json.loads(vesAgentConfigStr) + if vesAgentConfig is None: + logger.warn("VesAgentBacklogs.config.%s corrupts" % (vimid)) + return 0 + + # iterate all backlog and remove the associate state! + # tbd + + # clear the whole backlogs for a VIM + cache.set("VesAgentBacklogs.config.%s" % vimid, "deleting the backlogs", 1) + + except Exception as e: + self._logger.error("exception:%s" % str(e)) + + self._logger.info("return") + return 0 def buildBacklogsOneVIM(self, vimid, vesagent_config = None): ''' diff --git a/windriver/titanium_cloud/vesagent/vespublish.py b/windriver/titanium_cloud/vesagent/vespublish.py new file mode 100644 index 00000000..df77e30b --- /dev/null +++ b/windriver/titanium_cloud/vesagent/vespublish.py @@ -0,0 +1,43 @@ +# Copyright (c) 2017-2018 Wind River Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, unicode_literals + +import time +import logging +import json +import urllib2 + +logger = logging.getLogger(__name__) + +def publishAnyEventToVES(ves_subscription, event): + logger.info("Start to send single event to VES collector.") + endpoint = ves_subscription.get("endpoint", None) + username = ves_subscription.get("username", None) + password = ves_subscription.get("password", None) + + if endpoint: + try: + logger.info("publish event to VES: %s", ) + headers = {'Content-Type': 'application/json'} + request = urllib2.Request(url=endpoint, headers=headers, data=json.dumps(event)) + time.sleep(1) + response = urllib2.urlopen(request) + logger.info("VES response is: %s", response.read()) + except urllib2.URLError, e: + logger.critical("Failed to publish to %s: %s", endpoint, e.reason) + except Exception as e: + logger.error("exception:%s" % str(e)) + else: + logger.info("Missing VES info.")
\ No newline at end of file |