From 451a3400b76511393c62a444f588a4ed15f4a549 Mon Sep 17 00:00:00 2001 From: Michael Lando Date: Sun, 19 Feb 2017 10:28:42 +0200 Subject: Initial OpenECOMP SDC commit Change-Id: I0924d5a6ae9cdc161ae17c68d3689a30d10f407b Signed-off-by: Michael Lando --- .../sdc-elasticsearch/attributes/default.rb | 1 + .../files/default/dashboard_BI-Dashboard.json | 13 ++ .../default/dashboard_Monitoring-Dashboared.json | 15 ++ .../sdc-elasticsearch/files/default/logging.yml | 23 ++ .../files/default/visualization_JVM-used-CPU.json | 10 + .../default/visualization_JVM-used-Memory.json | 10 + .../visualization_JVM-used-Threads-Num.json | 10 + ...ied-services-ampersand-resources-(per-day).json | 10 + ...ed-Resources-slash-Services-slash-Products.json | 10 + ...isualization_Show-all-distributed-services.json | 10 + .../files/default/visualization_host-used-CPU.json | 10 + .../visualization_host-used-Threads-Num.json | 10 + .../visualization_number-of-user-accesses.json | 10 + .../recipes/ES_1_setup_elasticsearch.rb | 16 ++ .../recipes/ES_2_setup_logging.rb | 6 + .../recipes/ES_3_create_audit_template.rb | 238 +++++++++++++++++++++ .../recipes/ES_4_create_resources_template.rb | 37 ++++ .../recipes/ES_5_create_monitoring_template.rb | 47 ++++ .../ES_6_kibana_dashboard_virtualization.rb | 57 +++++ .../templates/default/ES-elasticsearch.yml.erb | 16 ++ 20 files changed, 559 insertions(+) create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/attributes/default.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_BI-Dashboard.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_Monitoring-Dashboared.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/logging.yml create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-CPU.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Memory.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Threads-Num.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-certified-services-ampersand-resources-(per-day).json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-created-Resources-slash-Services-slash-Products.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-distributed-services.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-CPU.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-Threads-Num.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_number-of-user-accesses.json create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_2_setup_logging.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_3_create_audit_template.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_4_create_resources_template.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_kibana_dashboard_virtualization.rb create mode 100644 sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb (limited to 'sdc-os-chef/sdc-elasticsearch/chef-repo') diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/attributes/default.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/attributes/default.rb new file mode 100644 index 0000000000..4287ca8617 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/attributes/default.rb @@ -0,0 +1 @@ +# \ No newline at end of file diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_BI-Dashboard.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_BI-Dashboard.json new file mode 100644 index 0000000000..c740a94f5d --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_BI-Dashboard.json @@ -0,0 +1,13 @@ +{ + "title": "BI Dashboard", + "hits": 0, + "description": "", + "panelsJSON": "[{\"col\":1,\"id\":\"Show-all-certified-services-ampersand-resources-(per-day)\",\"panelIndex\":1,\"row\":1,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"col\":7,\"id\":\"Show-all-distributed-services\",\"panelIndex\":2,\"row\":1,\"size_x\":6,\"size_y\":5,\"type\":\"visualization\"},{\"id\":\"Show-all-created-Resources-slash-Services-slash-Products\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":6,\"size_y\":4,\"col\":1,\"row\":6},{\"id\":\"number-of-user-accesses\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":6,\"size_y\":4,\"col\":7,\"row\":6}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{}", + "version": 1, + "timeRestore": false, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_Monitoring-Dashboared.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_Monitoring-Dashboared.json new file mode 100644 index 0000000000..41b9c00c6e --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/dashboard_Monitoring-Dashboared.json @@ -0,0 +1,15 @@ +{ + "title": "Monitoring Dashboared", + "hits": 0, + "description": "", + "panelsJSON": "[{\"id\":\"JVM-used-CPU\",\"type\":\"visualization\",\"panelIndex\":1,\"size_x\":6,\"size_y\":5,\"col\":1,\"row\":5},{\"id\":\"JVM-used-Memory\",\"type\":\"visualization\",\"panelIndex\":2,\"size_x\":6,\"size_y\":4,\"col\":1,\"row\":1},{\"id\":\"JVM-used-Threads-Num\",\"type\":\"visualization\",\"panelIndex\":3,\"size_x\":6,\"size_y\":4,\"col\":7,\"row\":1},{\"id\":\"host-used-CPU\",\"type\":\"visualization\",\"panelIndex\":4,\"size_x\":3,\"size_y\":2,\"col\":7,\"row\":5},{\"id\":\"host-used-Memory\",\"type\":\"visualization\",\"panelIndex\":5,\"size_x\":3,\"size_y\":2,\"col\":10,\"row\":5},{\"id\":\"host-used-Threads-Num\",\"type\":\"visualization\",\"panelIndex\":6,\"size_x\":3,\"size_y\":2,\"col\":7,\"row\":7}]", + "optionsJSON": "{\"darkTheme\":false}", + "uiStateJSON": "{}", + "version": 1, + "timeRestore": true, + "timeTo": "now", + "timeFrom": "now-24h", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}}]}" + } +} \ No newline at end of file diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/logging.yml b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/logging.yml new file mode 100644 index 0000000000..a78fb3c0e3 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/logging.yml @@ -0,0 +1,23 @@ +# you can override this using by setting a system property, for example -Des.logger.level=DEBUG +es.logger.level: INFO +rootLogger: ${es.logger.level}, console +logger: + # log action execution errors for easier debugging + action: DEBUG + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + +appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n" diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-CPU.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-CPU.json new file mode 100644 index 0000000000..45565cf0a4 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-CPU.json @@ -0,0 +1,10 @@ +{ + "title": "JVM used CPU", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"jvmcpu\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"jvmid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"hostid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"monitoring_events-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} \ No newline at end of file diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Memory.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Memory.json new file mode 100644 index 0000000000..c030df4655 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Memory.json @@ -0,0 +1,10 @@ +{ + "title": "JVM used Memory", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"jvmmem\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"jvmid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"hostid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"monitoring_events-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Threads-Num.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Threads-Num.json new file mode 100644 index 0000000000..28d0468e9d --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_JVM-used-Threads-Num.json @@ -0,0 +1,10 @@ +{ + "title": "JVM used Threads Num", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"jvmtnum\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"jvmid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"hostid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"monitoring_events-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-certified-services-ampersand-resources-(per-day).json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-certified-services-ampersand-resources-(per-day).json new file mode 100644 index 0000000000..7d7ce6e4d4 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-certified-services-ampersand-resources-(per-day).json @@ -0,0 +1,10 @@ +{ + "title": "Show all certified services & resources (per day)", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"TIMESTAMP\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditingevents-*\",\"query\":{\"query_string\":{\"query\":\"ACTION:\\\"CertificationSuccess\\\" AND (RESOURCE_TYPE:\\\"Resource\\\" OR RESOURCE_TYPE:\\\"Service\\\")\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-created-Resources-slash-Services-slash-Products.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-created-Resources-slash-Services-slash-Products.json new file mode 100644 index 0000000000..37e06ca3ca --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-created-Resources-slash-Services-slash-Products.json @@ -0,0 +1,10 @@ +{ + "title": "Show all created Resources/Services/Products", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"TIMESTAMP\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"RESOURCE_TYPE\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditingevents-*\",\"query\":{\"query_string\":{\"query\":\"ACTION:\\\"Create\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-distributed-services.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-distributed-services.json new file mode 100644 index 0000000000..5d375733f5 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_Show-all-distributed-services.json @@ -0,0 +1,10 @@ +{ + "title": "Show all distributed services", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"TIMESTAMP\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditingevents-*\",\"query\":{\"query_string\":{\"query\":\"ACTION:\\\"DResult\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-CPU.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-CPU.json new file mode 100644 index 0000000000..3c9f7d5425 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-CPU.json @@ -0,0 +1,10 @@ +{ + "title": "host used CPU", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"hostcpu\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"h\",\"customInterval\":\"24h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"hostid\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"monitoring_events-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-Threads-Num.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-Threads-Num.json new file mode 100644 index 0000000000..a6a7c09395 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_host-used-Threads-Num.json @@ -0,0 +1,10 @@ +{ + "title": "host used Threads Num", + "visState": "{\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"jvmtnum\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"timestamp\",\"interval\":\"h\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"hostid\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"monitoring_events-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_number-of-user-accesses.json b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_number-of-user-accesses.json new file mode 100644 index 0000000000..fdcd88c16f --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/files/default/visualization_number-of-user-accesses.json @@ -0,0 +1,10 @@ +{ + "title": "number of user accesses", + "visState": "{\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"TIMESTAMP\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"auditingevents-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"ACTION:\\\"Access\\\"\"}},\"filter\":[]}" + } +} diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb new file mode 100644 index 0000000000..854898c1b7 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb @@ -0,0 +1,16 @@ +clusterName = node['elasticsearch'][:cluster_name]+node.chef_environment +node_name = node[:hostname] + +template "/usr/share/elasticsearch/config/elasticsearch.yml" do + source "ES-elasticsearch.yml.erb" + owner "elasticsearch" + group "elasticsearch" + mode "0755" + variables({ + :cluster_name => "#{clusterName}", + :node_name => node_name, + :ES_IP => node['Nodes']['ES'], + :num_of_shards => node['elasticsearch'][:num_of_shards], + :num_of_replicas => node['elasticsearch'][:num_of_replicas] + }) +end diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_2_setup_logging.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_2_setup_logging.rb new file mode 100644 index 0000000000..5e462966ed --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_2_setup_logging.rb @@ -0,0 +1,6 @@ +cookbook_file "/usr/share/elasticsearch/config/logging.yml" do + source "logging.yml" + owner "elasticsearch" + group "elasticsearch" + mode "0755" +end diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_3_create_audit_template.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_3_create_audit_template.rb new file mode 100644 index 0000000000..2d882631f2 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_3_create_audit_template.rb @@ -0,0 +1,238 @@ +ruby_block "check_ElasticSearch_Cluster_Health" do + block do + #tricky way to load this Chef::Mixin::ShellOut utilities + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + #curl_command = "http://#{node['ipaddress']}:9200/_cluster/health?pretty=true" + curl_command = "http://localhost:9200/_cluster/health?pretty=true" + resp = Net::HTTP.get_response URI.parse(curl_command) + stat = JSON.parse(resp.read_body)['status'] + + case stat + when "green" + printf("\033[32m%s\n\033[0m", " ElasticSearch Cluster status is green.") + when "yellow" + printf("\033[33m%s\n\033[0m", " ElasticSearch Cluster status is yellow...") + when "red" + printf("\033[31m%s\n\033[0m", " ElasticSearch Cluster status is red!") + end + end + retries 10 + retry_delay 2 +end + + +bash "create audit mapping" do + code <<-EOH + curl -i -X PUT -d '{ "order": 1, "template": "auditingevents-*", "settings": {}, "mappings": + { + "distributiondownloadevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "RESOURCE_URL": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CONSUMER_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "auditinggetuebclusterevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CONSUMER_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "distributionstatusevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "RESOURCE_URL": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TOPIC_NAME":{ "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CONSUMER_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "distributionengineevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TOPIC_NAME":{ "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ROLE": { "include_in_all": true, "type": "string" }, + "API_KEY": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "D_ENV": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CONSUMER_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "useraccessevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "USER": { "include_in_all": true, "type": "string" } + }, + "_all": { "enabled": true } + }, + "resourceadminevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "INVARIANT_UUID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_VERSION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_STATE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "PREV_VERSION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "PREV_STATE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "RESOURCE_NAME": { "include_in_all": true, "type": "string" }, + "RESOURCE_TYPE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DPREV_STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DCURR_STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TOSCA_NODE_TYPE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "COMMENT": { "include_in_all": true, "type": "string" }, + "ARTIFACT_DATA": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "PREV_ARTIFACT_UUID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_ARTIFACT_UUID": { "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "useradminevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "USER_AFTER": { "include_in_all": true, "type": "string" }, + "USER_BEFORE": { "include_in_all": true, "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" } + }, + "_all": { "enabled": true } + }, + "distributionnotificationevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_STATE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_VERSION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "RESOURCE_NAME": { "include_in_all": true, "type": "string" }, + "RESOURCE_TYPE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TOPIC_NAME":{ "include_in_all": true, "index": "not_analyzed", "type": "string" } + }, + "_all": { "enabled": true } + }, + "categoryevent": { + "properties": { + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CATEGORY_NAME": { "include_in_all": true, "type": "string" }, + "SUB_CATEGORY_NAME": { "include_in_all": true, "type": "string" }, + "GROUPING_NAME": { "include_in_all": true, "type": "string" }, + "RESOURCE_TYPE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } + }, + "authevent": { + "properties": { + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" }, + "DESC": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "URL": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "USER": { "include_in_all": true, "type": "string" }, + "AUTH_STATUS": { "include_in_all": true, "index": "not_analyzed","type": "string" } , + "REALM": { "include_in_all": true, "index": "not_analyzed","type": "string" } + }, + "_all": { "enabled": true } + }, + "consumerevent": { + "properties": { + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "ECOMP_USER": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } + }, + "getuserslistevent": { + "properties": { + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DETAILS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } + }, + "getcategoryhierarchyevent": { + "properties": { + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DETAILS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } + }, + "distributiondeployevent": { + "properties": { + "ACTION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "CURR_VERSION": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "DESC": { "include_in_all": true, "type": "string" }, + "DID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "MODIFIER": { "include_in_all": true, "type": "string" }, + "REQUEST_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "RESOURCE_NAME": { "include_in_all": true, "type": "string" }, + "RESOURCE_TYPE": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "SERVICE_INSTANCE_ID": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "STATUS": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "TIMESTAMP": { "include_in_all": true, "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } } + }, + "aliases": { "last_3_months": {}}}' http://localhost:9200/_template/audit_template + EOH +end diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_4_create_resources_template.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_4_create_resources_template.rb new file mode 100644 index 0000000000..4dc264153d --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_4_create_resources_template.rb @@ -0,0 +1,37 @@ +ruby_block "check_ElasticSearch_Cluster_Health" do + block do + #tricky way to load this Chef::Mixin::ShellOut utilities + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + #curl_command = "http://#{node['ipaddress']}:9200/_cluster/health?pretty=true" + curl_command = "http://localhost:9200/_cluster/health?pretty=true" + resp = Net::HTTP.get_response URI.parse(curl_command) + stat = JSON.parse(resp.read_body)['status'] + + case stat + when "green" + printf("\033[32m%s\n\033[0m", " ElasticSearch Cluster status is green.") + when "yellow" + printf("\033[33m%s\n\033[0m", " ElasticSearch Cluster status is yellow...") + when "red" + printf("\033[31m%s\n\033[0m", " ElasticSearch Cluster status is red!") + end + end + retries 10 + retry_delay 2 +end + +bash "create resources mapping" do + code <<-EOH + curl -i -X PUT -d '{ "order": 1, "template": "resources", "settings": {}, "mappings": + { + "esartifactdata": { + "properties": { + "id": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "data": { "include_in_all": false, "type": "string" } + }, + "_all": { "enabled": true } + } + } + }' http://localhost:9200/_template/resources_template + EOH +end \ No newline at end of file diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb new file mode 100644 index 0000000000..dfb68c1dfa --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb @@ -0,0 +1,47 @@ +ruby_block "check_ElasticSearch_Cluster_Health" do + block do + #tricky way to load this Chef::Mixin::ShellOut utilities + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + #curl_command = "http://#{node['ipaddress']}:9200/_cluster/health?pretty=true" + curl_command = "http://localhost:9200/_cluster/health?pretty=true" + resp = Net::HTTP.get_response URI.parse(curl_command) + stat = JSON.parse(resp.read_body)['status'] + + case stat + when "green" + printf("\033[32m%s\n\033[0m", " ElasticSearch Cluster status is green.") + when "yellow" + printf("\033[33m%s\n\033[0m", " ElasticSearch Cluster status is yellow...") + when "red" + printf("\033[31m%s\n\033[0m", " ElasticSearch Cluster status is red!") + end + end + retries 10 + retry_delay 2 +end + +bash "create monitoring mapping" do + code <<-EOH + curl -i -X PUT -d '{ "order": 1, "template": "monitoring_events-*", "settings": {}, "mappings": + { + "monitoringevent": { + "properties": { + "hostid": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "hostcpu": { "include_in_all": true, "type": "long" }, + "hostmem": { "include_in_all": true, "type": "double" }, + "hostdisk": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "jvmid": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "jvmcpu": { "include_in_all": true,"type": "long" }, + "jvmmem": { "include_in_all": true, "type": "long" }, + "jvmtnum": { "include_in_all": true, "type": "integer" }, + "appid": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "appstat": { "include_in_all": true, "index": "not_analyzed", "type": "string" }, + "timestamp": { "include_in_all": true, "index": "not_analyzed", "ignore_malformed": false, "format": "yyyy-MM-dd HH:mm:ss.SSS z", "precision_step": 4, "type": "date" } + }, + "_all": { "enabled": true } + } + }, + "aliases": { "last_3_months": {} } + }' http://localhost:9200/_template/monitoring_template + EOH +end diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_kibana_dashboard_virtualization.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_kibana_dashboard_virtualization.rb new file mode 100644 index 0000000000..a3b15073a1 --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_kibana_dashboard_virtualization.rb @@ -0,0 +1,57 @@ +cookbook_file "/usr/share/elasticsearch/config/kibana_dashboard_virtualization.json" do + source "kibana_dashboard_virtualization.json" + owner "elasticsearch" + group "elasticsearch" + mode "0755" +end + + + +ruby_block "check_ElasticSearch_Cluster_Health" do + block do + #tricky way to load this Chef::Mixin::ShellOut utilities + Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut) + #curl_command = "http://#{node['ipaddress']}:9200/_cluster/health?pretty=true" + curl_command = "http://localhost:9200/_cluster/health?pretty=true" + resp = Net::HTTP.get_response URI.parse(curl_command) + stat = JSON.parse(resp.read_body)['status'] + + case stat + when "green" + printf("\033[32m%s\n\033[0m", " ElasticSearch Cluster status is green.") + when "yellow" + printf("\033[33m%s\n\033[0m", " ElasticSearch Cluster status is yellow...") + when "red" + printf("\033[31m%s\n\033[0m", " ElasticSearch Cluster status is red!") + end + end + retries 10 + retry_delay 2 +end + + +bash "create Kibana dashboard" do + code <<-EOH + for file in /root/chef-solo/cookbooks/sdc-elasticsearch/files/default/dashboard_*.json; do + name=`basename $file .json | awk -F"_" '{print $2}'` + echo "Loading dashboard $name:" + curl -XPUT http://localhost:9200/.kibana/dashboard/$name -d @$file || exit 1 + echo + done + EOH +end + + +bash "create Kibana visualization" do + code <<-EOH + for file in /root/chef-solo/cookbooks/sdc-elasticsearch/files/default/visualization_*.json; do + name=`basename $file .json | awk -F"_" '{print $2}'` + echo "Loading visualization $name:" + curl -XPUT http://localhost:9200/.kibana/visualization/$name -d @$file || exit 1 + echo + done + EOH +end + + + diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb new file mode 100644 index 0000000000..613e6b046b --- /dev/null +++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb @@ -0,0 +1,16 @@ +network.host: 0.0.0.0 +discovery.zen.ping.multicast.enabled: false +discovery.zen.ping.unicast.enabled: true +#discovery.zen.ping.unicast.hosts: <%= @ES_IP %> + +cluster.name: <%= @cluster_name %> +node.name: <%= @node_name %> +elasticSearch.transportclient: true + +transport.client.initial_nodes: + - <%= @ES_IP %>:9300 + +index.number_of_shards: <%= @num_of_shards %> +index.number_of_replicas: <%= @num_of_replicas %> + +path.logs: /usr/share/elasticsearch/logs -- cgit 1.2.3-korg