diff options
Diffstat (limited to 'extra/docker/elk')
18 files changed, 851 insertions, 0 deletions
diff --git a/extra/docker/elk/.env b/extra/docker/elk/.env new file mode 100644 index 0000000..79d7943 --- /dev/null +++ b/extra/docker/elk/.env @@ -0,0 +1,2 @@ +# docker compose related +COMPOSE_PROJECT_NAME=clamp_elk diff --git a/extra/docker/elk/README.md b/extra/docker/elk/README.md new file mode 100644 index 0000000..b2b6fe3 --- /dev/null +++ b/extra/docker/elk/README.md @@ -0,0 +1,90 @@ +CLAMP Dashboard +=============== + +ELK stack for CLAMP : Logstash is used to retrieve events and notifications from DMaaP and pushes them into Elasticsearch. +Kibana is then used to extract statistics. + + +Deployment instructions +----------------------- + +Requirements: docker-compose + +1. Update configuration in docker-compose file +2. `docker-compose up -d elasticsearch logstash kibana` + +If you encounter connection problems with kibana, first deploy elasticsearch, wait for it to be available, then kibana. + +Backup/restore +-------------- + +backup.py and restore.py scripts are available inside the kibana docker image for saving and restoring the configuration. +the default configuration is located in the kibana docker image under the directory "/saved-objects/default/" + +### backup.py +``` +firts create the directory "/saved-objects/mybackup" if it doesn't exist +docker-compose exec kibana backup.py -C /saved-objects/mybackup +``` +``` +usage: backup.py [-h] [-v] [-C CONFIGURATION_PATH] [-f] [-H KIBANA_HOST] + +Description of the script + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Use verbose logging + -C CONFIGURATION_PATH, --configuration_path CONFIGURATION_PATH + Path of the configuration to be backed up. + -f, --force If the save folder already exists, overwrite files + matching a configuration item that should be written. + Files already in the folder that do not match are left + as-is. + -H KIBANA_HOST, --kibana-host KIBANA_HOST + Kibana endpoint. + +``` + +### restore.py +``` +to restore the configuration, you previously backed up, use the command below: +docker-compose exec kibana restore.py -C /saved-objects/mybackup -f +to restore the default confgiuration use the command below: +docker-compose exec kibana restore.py -C /saved-objects/default -f +``` +``` +usage: restore.py [-h] [-v] [-C CONFIGURATION_PATH] [-H KIBANA_HOST] [-f] + +Restores the kibana configuration. + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Use verbose logging + -C CONFIGURATION_PATH, --configuration_path CONFIGURATION_PATH + Path of the configuration to be restored.Should + contain at least one folder named index- + pattern,config,search,visualization or dashboard + -H KIBANA_HOST, --kibana-host KIBANA_HOST + Kibana endpoint. + -f, --force Overwrite configuration if needed. +``` + +Tools +----- + +The following tools are available in the 'tools/' folder. + + +### EsAutoQuery + +Small script ease Elasticsearch /painless/ field development. +It reads a json file as a query for Elasticsearch, pushes it on the ES server, and display back the answer in a loop, each time the file is modified. + + +### DMaaP Service Mocker + +Script that simulates control loop DMaaP services to provide sample data to logstash through DMaaP. + +TODO +---- +* Add a script that verifies that elasticsearch is available before starting loading the default configuration for kibana diff --git a/extra/docker/elk/docker-compose.yml b/extra/docker/elk/docker-compose.yml new file mode 100644 index 0000000..fe7fa06 --- /dev/null +++ b/extra/docker/elk/docker-compose.yml @@ -0,0 +1,66 @@ +version: '3.1' + +services: + elasticsearch: + image: onap/clamp-dashboard-elasticsearch:latest + volumes: + - ../../../src/main/docker/elasticsearch/config/ca-certs.pem:/usr/share/elasticsearch/config/root-ca.pem + - ../../../src/main/docker/elasticsearch/config/clamp.pem:/usr/share/elasticsearch/config/esnode.pem + - ../../../src/main/docker/elasticsearch/config/clamp-key.pem:/usr/share/elasticsearch/config/esnode-key.pem + - ../../../src/main/docker/elasticsearch/config/clamp.pem:/usr/share/elasticsearch/config/kirk.pem + - ../../../src/main/docker/elasticsearch/config/clamp-key.pem:/usr/share/elasticsearch/config/kirk-key.pem + - ../../../src/main/docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + ports: + - 9200:9200 + networks: + es_net: + aliases: + - elasticsearch + environment: + - cluster.name=docker-cluster + - bootstrap.memory_lock=false + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + + logstash: + image: onap/clamp-dashboard-logstash:latest + build: + context: ../../../src/main/docker/logstash + dockerfile: Dockerfile.logstash + volumes: + - ../../../src/main/docker/logstash/pipeline:/usr/share/logstash/pipeline + - ./logstash-input:/log-input + depends_on: + - elasticsearch + networks: + es_net: + environment: + - dmaap_base_url=http://dmaapmocker:3904/ + - dmaap_user=user + - dmaap_password=password + - dmaap_consumer_group=clampdashboard + - dmaap_consumer_id=clampdashboard + - event_topic=DCAE-CL-EVENT + - notification_topic=POLICY-CL-MGT + - request_topic=APPC-CL + - elasticsearch_base_url=elasticsearch + - "LOGSTASH_USR=logstash" + - "LOGSTASH_PWD=logstash" + + kibana: + image: onap/clamp-dashboard-kibana:latest + build: + context: ../../../src/main/docker/kibana + dockerfile: Dockerfile.kibana + ports: + - 5601:5601 + depends_on: + - elasticsearch + volumes: + - ../../../src/main/docker/kibana/saved-objects/:/saved-objects/ + - ../../../src/main/docker/kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml + - ../../../src/main/docker/kibana/conf/keystore:/usr/share/kibana/config/keystore + networks: + es_net: + +networks: + es_net: diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py b/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py new file mode 100755 index 0000000..bd6caec --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +import os +import json +import copy +import random +import requests +import uuid +import time +from datetime import datetime + +def luck(n=2): + """ gives 1 chance out of n (default: 2) to return True """ + assert n > 1 + return bool(random.randint(0, n-1)) +def now_dmaap_timestamp(): + return str(datetime.now().timestamp()).replace(".","")[:13] +def now_notification_time(): + return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f+00:00") + +CONTROL_LOOP_NAMES = [ + 'CL-vCPE-d925ed73', + 'CL-vCPE-37b1c91e', + 'CL-vCPE-c2597657', + 'CL-vCPE-a11318ba', + 'CL-vCPE-5321c558', +] + +TEMPLATES = { + 'event_abated' :'event_abated.json', + 'event_onset' :'event_onset.json', + 'notification_active' :'notification_active.json', + 'notification_final_failed' :'notification_final_failed.json', + 'notification_final_open' :'notification_final_open.json', + 'notification_final_success' :'notification_final_success.json', + 'notification_operation_failure' :'notification_operation_failure.json', + 'notification_operation' :'notification_operation.json', + 'notification_operation_success' :'notification_operation_success.json', + 'notification_rejected_disabled' :'notification_rejected_disabled.json', + 'notification_rejected_missing' :'notification_rejected_missing.json', +} + +ERROR_MESSAGES = [ + ('APPC', 'APPC1 : timeout on restart','RESTART'), + ('APPC', 'APPC2 : cannot restart','RESTART'), + ('SO', 'SO1 : scale up failed', 'SCALEUP'), + ('SO', 'SO2 : scale down failed', 'SCALEDOWN'), +] + +for key in TEMPLATES: + with open(TEMPLATES[key]) as f: + content = f.read() + TEMPLATES[key] = json.loads(content) + + +class DMaaPMessage(dict): + + dmaap_host_url = "http://dmaap.host.url:9200/" + dmaap_username = None + dmaap_password = None + + @classmethod + def from_template(cls, tmpl, **kwargs): + obj = cls() + obj.update(copy.deepcopy(TEMPLATES[tmpl])) + for keys,value in kwargs.items(): + current_node = obj + keys = keys.split(".") + key = keys[0] + for i in range(len(keys) - 1): + current_node = current_node[keys[i]] + key = keys[i] + current_node[key] = value + return obj + + def publish(self, topic): + url = "%s/events/%s" % (self.dmaap_host_url, topic) + auth = None + if self.dmaap_username and self.dmaap_password: + auth = (self.dmaap_username, self.dmaap_password) + response = requests.post(url, data=json.dumps(self), auth=auth) + return response.status_code + +class Event(DMaaPMessage): + + topic = "DCAE-CL-EVENT" + + @staticmethod + def abated(**kwargs): + return Event.from_template('event_abated', **kwargs) + + @staticmethod + def onset(**kwargs): + return Event.from_template('event_onset', **kwargs) + + def publish(self): + return super().publish(self.topic) + + +class Notification(DMaaPMessage): + + topic = "POLICY-CL-MGT" + + @classmethod + def from_template(cls, tmpl, **kwargs): + kwargs['notificationTime'] = now_notification_time() + return super().from_template(tmpl, **kwargs) + + @staticmethod + def active(**kwargs): + return Notification.from_template('notification_active', **kwargs) + + @staticmethod + def final(**kwargs): + class FinalNotification(Notification): + @staticmethod + def success(**kwargs): + return FinalNotification.from_template('notification_final_success', **kwargs) + @staticmethod + def failed(**kwargs): + msg = FinalNotification.from_template('notification_final_failed', **kwargs) + error = ERROR_MESSAGES[random.randint(0, len(ERROR_MESSAGES) - 1)] + h = msg['history'][-1] + h['actor'],h['message'],h['operation'] = error[0],error[1],error[2] + return msg + @staticmethod + def open(**kwargs): + return FinalNotification.from_template('notification_final_open', **kwargs) + return FinalNotification + + @staticmethod + def operation(**kwargs): + class OperationNotification(Notification): + @staticmethod + def success(**kwargs): + return OperationNotification.from_template('notification_operation_success', **kwargs) + @staticmethod + def failure(**kwargs): + return OperationNotification.from_template('notification_operation_failure', **kwargs) + return OperationNotification.from_template('notification_operation', **kwargs) + + @staticmethod + def rejected(**kwargs): + class RejectedNotification(Notification): + @staticmethod + def disabled(**kwargs): + return RejectedNotification.from_template('notification_rejected_disabled', **kwargs) + @staticmethod + def missing_fields(**kwargs): + return RejectedNotification.from_template('notification_rejected_missing', **kwargs) + + return RejectedNotification + + def publish(self): + return super().publish(self.topic) + + + +class CLStatus(object): + + def __init__(self, dmaap_url=None, + missing=None, disabled=None, op_failure=None): + self._stopped = False + def maybe(thing, ): + if thing is None: + thing = not luck(10) + return thing + self._missing = maybe(missing) + self._disabled = maybe(disabled) + self._op_failure = maybe(op_failure) + self._config = dict( + requestID=str(uuid.uuid4()), + closedLoopControlName=CONTROL_LOOP_NAMES[random.randint(0, len(CONTROL_LOOP_NAMES) - 1)] + ) + + def __iter__(self): + return next(self) + + def __next__(self): + if self._stopped: + raise StopIteration() + config = self._config + config.update(dict(closedLoopAlarmStart=now_dmaap_timestamp())) + yield Event.onset(**config) + if self._missing: + self._stopped = True + yield Notification.rejected().missing_fields(**config) + raise StopIteration() + elif self._disabled: + self._stopped = True + yield Notification.rejected().disabled(**config) + raise StopIteration() + + yield Notification.active(**config) + yield Notification.operation(**config) + + config['closedLoopAlarmEnd'] = now_dmaap_timestamp() + if self._op_failure: + yield Notification.operation().failure(**config) + self._stopped = True + yield Notification.final().failed(**config) + else: + yield Notification.operation().success(**config) + yield Event.abated(**config) + self._stopped = True + yield Notification.final().success(**config) + raise StopIteration() + +def print_usage(): + print(""" + ./ds_mocker.py <DMAAP_URL> <EVENT_TOPIC> [NOTIFICATION_TOPIC [REQUEST_TOPIC]] + """) + exit() + +def push(test_datas): + for current_i, status in enumerate(test_datas): + time.sleep(random.randint(0,3)) + for s in status: + # print(s) + status_code = s.publish() + if status_code != 200: + print("Error when publishing : status_code={}".format(status_code)) + exit(1) + time.sleep(random.randint(0,3)) + print("%03d,missing:%5s,disabled:%5s,op_failure:%5s - %s" % (current_i, status._missing, status._disabled, status._op_failure, status._config)) + + + +def generate_dataset_1(): + test_datas = [CLStatus(missing=False, disabled=False, op_failure=False) for i in range(300)] \ + + [CLStatus(missing=True, disabled=False, op_failure=False) for i in range(5)] \ + + [CLStatus(missing=False, disabled=True, op_failure=False) for i in range(6)] \ + + [CLStatus(missing=False, disabled=False, op_failure=True) for i in range(12)] + random.shuffle(test_datas) + return test_datas + +def generate_error_dataset_1(): + test_datas = [CLStatus(missing=False, disabled=False, op_failure=True) for i in range(60)] + random.shuffle(test_datas) + return test_datas + + +DATASETS = { + 'dataset_1': generate_dataset_1, + 'op_failure_1': generate_error_dataset_1, +} + +if __name__ == "__main__": + import sys + if len(sys.argv) < 3: + print_usage() + + DMaaPMessage.dmaap_host_url = sys.argv[1] + Event.topic = sys.argv[2] + Notification.topic = len(sys.argv) > 3 and sys.argv[3] or sys.argv[2] + # Request.topic = len(sys.argv) > 4 or Notification.topic + #push(DATASETS['op_failure_1']()) + push(DATASETS['dataset_1']()) diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/event_abated.json b/extra/docker/elk/tools/DMaaPServiceMocker/event_abated.json new file mode 100644 index 0000000..ae0d4b7 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/event_abated.json @@ -0,0 +1,27 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "closedLoopAlarmEnd": 1478710227625, + "closedLoopAlarmStart": 1478709727496, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "closedLoopEventStatus": "ABATED", + "from": "DCAE", + "policyName": "vFirewall", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/event_onset.json b/extra/docker/elk/tools/DMaaPServiceMocker/event_onset.json new file mode 100644 index 0000000..223f096 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/event_onset.json @@ -0,0 +1,26 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "closedLoopAlarmStart": 1478709385444, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "closedLoopEventStatus": "ONSET", + "from": "DCAE", + "policyName": "vFirewall", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "88e951ad-a07c-4748-9cef-34f32851a527", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_active.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_active.json new file mode 100644 index 0000000..341a249 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_active.json @@ -0,0 +1,28 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "notification": "ACTIVE", + "notificationTime": "2016-11-09 16:46:29.721000+00:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_failed.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_failed.json new file mode 100644 index 0000000..4891c7e --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_failed.json @@ -0,0 +1,42 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmEnd": 1478710489739, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "history": [ + { + "actor": "APPC", + "end": 1478637014434, + "message": "<APPC ERROR code/description>", + "operation": "RESTART", + "outcome": "FAILED", + "start": 1478637014430, + "subRequestId": "1", + "target": "vserver.vserver-name" + } + ], + "message": "actor=APPC,operation=RESTART,target=vserver.vserver-name,subRequestId=1,outcome=FAILED", + "notification": "FINAL: FAILURE", + "notificationTime": "2016-11-08 15:30:14.434000-05:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_open.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_open.json new file mode 100644 index 0000000..da9e6e1 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_open.json @@ -0,0 +1,28 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "closedLoopAlarmEnd": 1478710489739, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "notification": "FINAL: OPEN", + "notificationTime": "2016-11-08 15:30:14.434000-05:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_success.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_success.json new file mode 100644 index 0000000..3d027eb --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_final_success.json @@ -0,0 +1,40 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmEnd": 1478710489739, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "history": [ + { + "actor": "APPC", + "end": 1478709989738, + "operation": "RESTART", + "start": 1478709989734, + "subRequestId": "1", + "target": "vserver.vserver-name" + } + ], + "message": "actor=APPC,operation=RESTART,target=vserver.vserver-name,subRequestId=1", + "notification": "FINAL: SUCCESS", + "notificationTime": "2016-11-09 11:46:29.738000-05:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation.json new file mode 100644 index 0000000..341a249 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation.json @@ -0,0 +1,28 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "notification": "ACTIVE", + "notificationTime": "2016-11-09 16:46:29.721000+00:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_failure.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_failure.json new file mode 100644 index 0000000..5dbe33c --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_failure.json @@ -0,0 +1,29 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "message": "<APPC ERROR code and/or message>", + "notification": "OPERATION: FAILURE", + "notificationTime": "2016-11-08 15:30:14.434000-05:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_success.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_success.json new file mode 100644 index 0000000..9e18b97 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_operation_success.json @@ -0,0 +1,29 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "false", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "message": "actor=APPC,operation=RESTART,target=vserver.vserver-name,subRequestId=1", + "notification": "OPERATION: SUCCESS", + "notificationTime": "2016-11-09 11:46:29.738000-05:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_disabled.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_disabled.json new file mode 100644 index 0000000..3d94987 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_disabled.json @@ -0,0 +1,29 @@ +{ + "AAI": { + "cloud-region.identity-url": "http://1.2.3.4:5000/v2.0", + "complex.physical-location-id": "SNYSCA12", + "generic-vnf.service-id": "eb477a13-f380-401e-b2bd-f2d444c63e54", + "generic-vnf.vnf-name": "gupi1337p", + "tenant.tenant-id": "8f7b9a8a-8607-4a01-a5ef-0b26f569dcbb", + "vserver.is-closed-loop-disabled": "true", + "vserver.prov-status": "PROV", + "vserver.vserver-selflink": "https//2.3.4.5:1234/d5a992de-8a70-4d7d-b382-2bc6eea52317", + "vserver.vserver-id": "08f6c1f9-99e7-49f3-a662-c62b9f200d79", + "vserver.vserver-name": "gupi1337pm004" + }, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "message": "Closed Loop Disabled", + "notification": "REJECTED", + "notificationTime": "2016-05-11 15:09:23.896000+00:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_missing.json b/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_missing.json new file mode 100644 index 0000000..c901401 --- /dev/null +++ b/extra/docker/elk/tools/DMaaPServiceMocker/notification_rejected_missing.json @@ -0,0 +1,18 @@ +{ + "AAI": {}, + "OPS_CL_timer": 600, + "closedLoopAlarmStart": 1478709989621, + "closedLoopControlName": "ClosedLoop-3ffd062d-7589-43b2-865a-139b64e2d402", + "closedLoopEventClient": "DCAE_INSTANCE_ID.dcae-tca", + "from": "Policy", + "message": "Missing AAI tag", + "notification": "REJECTED", + "notificationTime": "2016-05-11 14:42:23.629000+00:00", + "policyName": "ClosedLoop.vUSP", + "policyScope": "resource=SampleResource,service=SampleService,type=SampleType,closedLoopControlName=SampleClosedLoop", + "policyVersion": "v0.0.1", + "requestID": "f889f60c-a715-48c0-801e-26428f6176b9", + "target": "vserver.vserver-name", + "target_type": "VM", + "version": "1.0.2" +} diff --git a/extra/docker/elk/tools/EsAutoQuery/autoQueryLoop.sh b/extra/docker/elk/tools/EsAutoQuery/autoQueryLoop.sh new file mode 100755 index 0000000..4929412 --- /dev/null +++ b/extra/docker/elk/tools/EsAutoQuery/autoQueryLoop.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +QUERY_FILE=${1:-query.json} +INDEX=${2:-logstash-*} +HOST_URL=${3:-http://localhost:9200} +URL=$HOST_URL/$INDEX/_search + +function usage() { + echo "Usage: $0 [QUERY_FILE [INDEX [HOST_URL]]]" + echo + echo "This script automatically sends the query file to elasticsearch" + echo "each time it's modified." +} + +if [ "${1}" == "--help" ]; +then + usage + exit 0 +fi + +echo "Querying '$URL' with '$QUERY_FILE'" +while [ 1 ]; +do + curl -XGET "$URL" -H 'Content-Type: application/json' -d"@$QUERY_FILE" | js-beautify + echo + inotifywait -e modify query.json +done diff --git a/extra/docker/elk/tools/EsAutoQuery/closedLoopAlarmDuration.json b/extra/docker/elk/tools/EsAutoQuery/closedLoopAlarmDuration.json new file mode 100644 index 0000000..5a29545 --- /dev/null +++ b/extra/docker/elk/tools/EsAutoQuery/closedLoopAlarmDuration.json @@ -0,0 +1,34 @@ +{ + "query" : { + "bool": { + "must": [ + { "match": { "closedLoopEventStatus": "ABATED" } } + ] + } + }, + "script_fields" : { + "closedLoopAlarmDuration" : { + "script" : { + "lang": "painless", + "source": " +if (doc.get('closedLoopEventStatus.keyword').value == 'ABATED') { + return doc.get('closedLoopAlarmEnd').value - doc.get('closedLoopAlarmStart').value; +} +return null +" + } + } + , "closedLoopAlarmStart" : { + "script" : { + "lang": "painless", + "source": "doc['closedLoopAlarmStart']" + } + } + , "closedLoopAlarmEnd" : { + "script" : { + "lang": "painless", + "source": "doc['closedLoopAlarmEnd']" + } + } + } +} diff --git a/extra/docker/elk/tools/EsAutoQuery/timeSince.json b/extra/docker/elk/tools/EsAutoQuery/timeSince.json new file mode 100644 index 0000000..6ee1493 --- /dev/null +++ b/extra/docker/elk/tools/EsAutoQuery/timeSince.json @@ -0,0 +1,51 @@ +{ + "query" : { + "match_all": {} + }, + "script_fields" : { + "timeSince" : { + "script" : { + "lang": "painless", + "source": " +long now = System.currentTimeMillis(); +if (doc.get('closedLoopEventStatus.keyword').value == 'ABATED') { + return now - doc.get('closedLoopAlarmEnd').value; +} +if (doc.get('closedLoopEventStatus.keyword').value == 'ONSET') { + return now - doc.get('closedLoopAlarmStart').value; +} +if (doc.containsKey('notification.keyword')) { + return now - doc.get('notificationTime').value; +} + +return null +" + } + } + , "closedLoopAlarmStart" : { + "script" : { + "lang": "painless", + "source": "doc['closedLoopAlarmStart']" + } + } + , "closedLoopEventStatus" : { + "script" : { + "lang": "painless", + "source": "doc['closedLoopEventStatus.keyword']" + } + } + , "notification" : { + "script" : { + "lang": "painless", + "source": "doc['notification.keyword']" + } + } + , "notificationTime" : { + "script" : { + "lang": "painless", + "source": "doc['notificationTime'].value" + } + } + + } +} |