Merge changes Ief706086,I8b33340f

* changes:
  Update missing fields
  Random error message when op_failed

2 files changed, 29 insertions, 6 deletions

diff --git a/extra/docker/elk/es_backup/default.json b/extra/docker/elk/es_backup/default.json
index 131af946c..fb31a41be 100644
--- a/extra/docker/elk/es_backup/default.json
+++ b/extra/docker/elk/es_backup/default.json
@@ -15,4 +15,4 @@
 {"_index":".kibana","_type":"search","_id":"AWIlAjTaDjJ4k3sPXRe1","_score":1,"_source":{"title":"FAILURE NOTIFICATION","description":"","columns":["closedLoopAlarmStart","closedLoopAlarmEnd","closedLoopControlName","closedLoopEventStatus","notification"],"sort":["closedLoopAlarmStart","desc"],"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"ABATED\",\"disabled\":true,\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"key\":\"closedLoopEventStatus.keyword\",\"negate\":false,\"type\":\"phrase\",\"value\":\"ABATED\"},\"query\":{\"match\":{\"closedLoopEventStatus.keyword\":{\"query\":\"ABATED\",\"type\":\"phrase\"}}}},{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"ONSET\",\"disabled\":true,\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"key\":\"closedLoopEventStatus.keyword\",\"negate\":false,\"type\":\"phrase\",\"value\":\"ONSET\"},\"query\":{\"match\":{\"closedLoopEventStatus.keyword\":{\"query\":\"ONSET\",\"type\":\"phrase\"}}}},{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"FAILURE NOTIFICATION\",\"disabled\":false,\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"key\":\"notification.keyword\",\"negate\":false,\"type\":\"phrase\",\"value\":\"FINAL: FAILURE\"},\"query\":{\"match\":{\"notification.keyword\":{\"query\":\"FINAL: FAILURE\",\"type\":\"phrase\"}}}}]}"}}}
 {"_index":".kibana","_type":"search","_id":"AWIk8yDIDjJ4k3sPXRbu","_score":1,"_source":{"title":"ONSET","description":"","columns":["closedLoopAlarmStart","closedLoopAlarmEnd","closedLoopControlName","closedLoopEventStatus","notification"],"sort":["closedLoopAlarmStart","desc"],"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"highlightAll\":true,\"version\":true,\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"closedLoopEventStatus.keyword\",\"value\":\"ONSET\"},\"query\":{\"match\":{\"closedLoopEventStatus.keyword\":{\"query\":\"ONSET\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"}}}
 {"_index":".kibana","_type":"dashboard","_id":"AWI-9KyytmDBG_mksMZL","_score":1,"_source":{"title":"CLAMP Dashboard","description":"","panelsJSON":"[{\"col\":5,\"id\":\"AWIk_VLEDjJ4k3sPXReS\",\"panelIndex\":1,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AWIlBLT7DjJ4k3sPXRe7\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWI967sW8kHy-lwNJZrR\",\"panelIndex\":3,\"row\":7,\"size_x\":5,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWI91Rvl8kHy-lwNJZrN\",\"panelIndex\":4,\"row\":4,\"size_x\":5,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWJoFDTyr1AwiIiFoV5R\",\"panelIndex\":5,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AWJoJdIjr1AwiIiFoV5T\",\"panelIndex\":6,\"row\":7,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"AWJoMx0_r1AwiIiFoV5V\",\"panelIndex\":7,\"row\":4,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":6,\"id\":\"AWJsT-HUr1AwiIiFoV6T\",\"panelIndex\":8,\"row\":4,\"size_x\":3,\"size_y\":6,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":9,\"type\":\"visualization\",\"id\":\"AWJxqD67B2zw2CQxMIeq\",\"col\":1,\"row\":10}]","optionsJSON":"{\"darkTheme\":false}","uiStateJSON":"{\"P-1\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-4\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-6\":{\"vis\":{\"legendOpen\":true}},\"P-8\":{\"vis\":{\"defaultColors\":{\"0 - 4\":\"rgb(0,104,55)\",\"4 - 6\":\"rgb(255,255,190)\",\"7 - 10\":\"rgb(165,0,38)\"},\"legendOpen\":false}},\"P-9\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}","timeRestore":false,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"}}}
-{"_index":".kibana","_type":"visualization","_id":"AWJoMx0_r1AwiIiFoV5V","_score":1,"_source":{"title":"CL Efficiency - Total Activity","visState":"{\"title\":\"CL Efficiency - Total Activity\",\"type\":\"line\",\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"closedLoopAlarmStart per 5 minutes\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":true},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Cumulative Sum of Sum of eventDuration\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Cumulative Sum of Sum of eventDuration\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cumulative_sum\",\"schema\":\"metric\",\"params\":{\"customMetric\":{\"id\":\"1-metric\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metricAgg\",\"params\":{\"field\":\"eventDuration\"}},\"metricAgg\":\"custom\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"closedLoopControlName.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"closedLoopAlarmStart\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":0,\"extended_bounds\":{}}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"query\":{\"match_all\":{}},\"filter\":[]}"}}}
+{"_index":".kibana","_type":"visualization","_id":"AWJoMx0_r1AwiIiFoV5V","_score":1,"_source":{"title":"CL Efficiency - Total Activity","visState":"{\"title\":\"CL Efficiency - Total Activity\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"closedLoopAlarmStart per 10 seconds\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Sum of eventDuration\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"cardinal\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Sum of eventDuration\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"eventDuration\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"closedLoopControlName.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"closedLoopAlarmStart\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":0,\"extended_bounds\":{}}}],\"listeners\":{}}","uiStateJSON":"{}","description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"query\":{\"match_all\":{}},\"filter\":[{\"meta\":{\"index\":\"AWIk7j2UDjJ4k3sPXRaY\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"closedLoopAlarmEnd\",\"value\":\"exists\"},\"exists\":{\"field\":\"closedLoopAlarmEnd\"},\"$state\":{\"store\":\"appState\"}}]}"}}}
diff --git a/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py b/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py
index 8710c3306..bd6caecce 100755
--- a/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py
+++ b/extra/docker/elk/tools/DMaaPServiceMocker/ds_mocker.py
@@ -39,6 +39,13 @@ TEMPLATES = {
     'notification_rejected_missing'  :'notification_rejected_missing.json',
 }
 
+ERROR_MESSAGES = [
+    ('APPC', 'APPC1 : timeout on restart','RESTART'),
+    ('APPC', 'APPC2 : cannot restart','RESTART'),
+    ('SO', 'SO1 : scale up failed', 'SCALEUP'),
+    ('SO', 'SO2 : scale down failed', 'SCALEDOWN'),
+]
+
 for key in TEMPLATES:
     with open(TEMPLATES[key]) as f:
         content = f.read()
@@ -110,7 +117,11 @@ class Notification(DMaaPMessage):
                 return FinalNotification.from_template('notification_final_success', **kwargs)
             @staticmethod
             def failed(**kwargs):
-                return FinalNotification.from_template('notification_final_failed', **kwargs)
+                msg = FinalNotification.from_template('notification_final_failed', **kwargs)
+                error = ERROR_MESSAGES[random.randint(0, len(ERROR_MESSAGES) - 1)]
+                h = msg['history'][-1]
+                h['actor'],h['message'],h['operation'] = error[0],error[1],error[2]
+                return msg
             @staticmethod
             def open(**kwargs):
                 return FinalNotification.from_template('notification_final_open', **kwargs)
@@ -149,7 +160,7 @@ class CLStatus(object):
     def __init__(self, dmaap_url=None,
                  missing=None, disabled=None, op_failure=None):
         self._stopped = False
-        def maybe(thing):
+        def maybe(thing, ):
             if thing is None:
                 thing = not luck(10)
             return thing
@@ -213,15 +224,26 @@ def push(test_datas):
         print("%03d,missing:%5s,disabled:%5s,op_failure:%5s - %s" % (current_i, status._missing, status._disabled, status._op_failure, status._config))
 
 
+
 def generate_dataset_1():
-    test_datas = [CLStatus(missing=False, disabled=False, op_failure=False) for i in range(45)]  \
+    test_datas = [CLStatus(missing=False, disabled=False, op_failure=False) for i in range(300)]  \
              + [CLStatus(missing=True, disabled=False, op_failure=False) for i in range(5)]  \
              + [CLStatus(missing=False, disabled=True, op_failure=False) for i in range(6)]  \
-             + [CLStatus(missing=False, disabled=False, op_failure=True) for i in range(7)]
+             + [CLStatus(missing=False, disabled=False, op_failure=True) for i in range(12)]
+    random.shuffle(test_datas)
+    return test_datas
+
+def generate_error_dataset_1():
+    test_datas = [CLStatus(missing=False, disabled=False, op_failure=True) for i in range(60)]
     random.shuffle(test_datas)
     return test_datas
 
 
+DATASETS = {
+    'dataset_1': generate_dataset_1,
+    'op_failure_1': generate_error_dataset_1,
+}
+
 if __name__ == "__main__":
     import sys
     if len(sys.argv) < 3:
@@ -231,4 +253,5 @@ if __name__ == "__main__":
     Event.topic = sys.argv[2]
     Notification.topic = len(sys.argv) > 3 and sys.argv[3] or sys.argv[2]
     # Request.topic = len(sys.argv) > 4 or Notification.topic
-    push(generate_dataset_1())
+    #push(DATASETS['op_failure_1']())
+    push(DATASETS['dataset_1']())