Add cluster resiliance support

Add configuration support for CS/ES cluster
and add truststore default files to support
ssl connection

Change-Id: Icc2831704e8a3c55ce9b380a39001891ce8d41ab
Issue-ID: SDC-1254
Signed-off-by: Areli, Fuss (af732p) <af732p@intl.att.com>

8 files changed, 45 insertions, 21 deletions

diff --git a/sdc-os-chef/environments/Template.json b/sdc-os-chef/environments/Template.json
index 9073f2c562..dae0860861 100644
--- a/sdc-os-chef/environments/Template.json
+++ b/sdc-os-chef/environments/Template.json
@@ -35,12 +35,16 @@
             "fqdn": ["10.0.11.1", "10.0.11.1"]
         },
         "Nodes": {
-            "CS":            "yyy",
+            "CS": [
+              "yyy"
+            ],
             "BE":            "yyy",
             "ONBOARDING_BE": "yyy",
             "FE":            "yyy",
-            "ES":            "yyy",
-            "KB":            "yyy"
+            "ES": [
+	       "yyy"
+            ],
+	    "KB":            "yyy"
         },
         "Plugins": {
             "DCAE": {
diff --git a/sdc-os-chef/scripts/k8s/kubernetes_run.sh b/sdc-os-chef/scripts/k8s/kubernetes_run.sh
index 7a73c2af3b..41797fcdca 100644
--- a/sdc-os-chef/scripts/k8s/kubernetes_run.sh
+++ b/sdc-os-chef/scripts/k8s/kubernetes_run.sh
@@ -90,7 +90,7 @@ status $?
 # SDC LOGS Tree    #
 ####################
 print_header "SDC - Deploy Pods ..."
-mkdir -p /dockerdata-nfs/onap/sdc/logs//BE/SDC/SDC-BE
-mkdir -p /dockerdata-nfs/onap/sdc/logs/FE/SDC/SDC-FE
-mkdir -p /dockerdata-nfs/onap/sdc/logs/ONBOARD/SDC/ONBOARD-BE
-chmod -R 777 /dockerdata-nfs/onap/sdc/logs/*
+sudo mkdir -p /dockerdata-nfs/onap/sdc/logs//BE/SDC/SDC-BE
+sudo mkdir -p /dockerdata-nfs/onap/sdc/logs/FE/SDC/SDC-FE
+sudo mkdir -p /dockerdata-nfs/onap/sdc/logs/ONBOARD/SDC/ONBOARD-BE
+sudo chmod -R 777 /dockerdata-nfs/onap/sdc/logs/*
diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb
index 68c1154ec6..f32e736f53 100644
--- a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb
+++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_1_setup_elasticsearch.rb
@@ -1,3 +1,10 @@
+elasticsearch_list = ''
+
+node['Nodes']['ES'].each  do |item|
+    elasticsearch_list += "- " + item + ":9300\n"
+end
+
+
 template "/usr/share/elasticsearch/config/elasticsearch.yml" do
    source "ES-elasticsearch.yml.erb"
    owner "elasticsearch"
@@ -6,7 +13,8 @@ template "/usr/share/elasticsearch/config/elasticsearch.yml" do
    variables({
         :cluster_name => node['elasticsearch'][:cluster_name]+node.chef_environment,
         :node_name => node[:hostname],
-        :ES_IP => node['Nodes']['ES'],
+        :es_ip_list => "#{elasticsearch_list}",
+        :es_ip_list_XXX => node['Nodes']['ES'],
         :num_of_shards => node['elasticsearch'][:num_of_shards],
         :num_of_replicas => node['elasticsearch'][:num_of_replicas]
    })
diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb
index 613e6b046b..0a2057dca6 100644
--- a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb
+++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/templates/default/ES-elasticsearch.yml.erb
@@ -1,14 +1,14 @@
 network.host: 0.0.0.0
 discovery.zen.ping.multicast.enabled: false
 discovery.zen.ping.unicast.enabled: true
-#discovery.zen.ping.unicast.hosts: <%= @ES_IP %>
+#discovery.zen.ping.unicast.hosts: <%= @es_ip_list_XXXXX %>
 
 cluster.name: <%= @cluster_name %>
 node.name: <%= @node_name %>
 elasticSearch.transportclient: true
 
 transport.client.initial_nodes:
-   - <%= @ES_IP %>:9300
+<%= @es_ip_list %>
 
 index.number_of_shards: <%= @num_of_shards %>
 index.number_of_replicas: <%= @num_of_replicas %>
diff --git a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_1_create_audit_template.rb b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_1_create_audit_template.rb
index 35bf49fb6d..8ee4087325 100644
--- a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_1_create_audit_template.rb
+++ b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_1_create_audit_template.rb
@@ -1,8 +1,11 @@
+# Get the first es node in the list
+es_node =  node['Nodes']['ES'].first
+
 ruby_block "check_ElasticSearch_Cluster_Health" do
     block do
 		#tricky way to load this Chef::Mixin::ShellOut utilities
 		Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut)
-		curl_command = "http://#{node['Nodes']['ES']}:9200/_cluster/health?pretty=true"
+		curl_command = "http://#{es_node}:9200/_cluster/health?pretty=true"
 		resp = Net::HTTP.get_response URI.parse(curl_command)
 		stat = JSON.parse(resp.read_body)['status']
 
@@ -233,13 +236,13 @@ bash "create audit mapping" do
 			},
             "_all": { "enabled": true } }
 		},
-        "aliases": { "last_3_months": {}}}' http://#{node['Nodes']['ES']}:9200/_template/audit_template
+        "aliases": { "last_3_months": {}}}' http://#{es_node}:9200/_template/audit_template
 	EOH
 end
 
 bash "set default index for Kibana" do
 	code <<-EOH
-        curl -XPUT http://#{node['Nodes']['ES']}:9200/.kibana/index-pattern/auditingevents-* -d '{"title" : "events-*",  "timeFieldName": "TIMESTAMP"}'
-        curl -XPUT http://#{node['Nodes']['ES']}:9200/.kibana/config/4.3.3 -d '{"defaultIndex" : "auditingevents-*"}'
+        curl -XPUT http://#{es_node}:9200/.kibana/index-pattern/auditingevents-* -d '{"title" : "events-*",  "timeFieldName": "TIMESTAMP"}'
+        curl -XPUT http://#{es_node}:9200/.kibana/config/4.3.3 -d '{"defaultIndex" : "auditingevents-*"}'
     EOH
 end
diff --git a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_2_create_resources_template.rb b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_2_create_resources_template.rb
index d35b81293e..11b8c9d060 100644
--- a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_2_create_resources_template.rb
+++ b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_2_create_resources_template.rb
@@ -1,8 +1,11 @@
+# Get the first es node in the list
+es_node =  node['Nodes']['ES'].first
+
 ruby_block "check_ElasticSearch_Cluster_Health" do
     block do
       #tricky way to load this Chef::Mixin::ShellOut utilities
       Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut)
-      curl_command = "http://#{node['Nodes']['ES']}:9200/_cluster/health?pretty=true"
+      curl_command = "http://#{es_node}:9200/_cluster/health?pretty=true"
       resp = Net::HTTP.get_response URI.parse(curl_command)
       stat = JSON.parse(resp.read_body)['status']
 
@@ -32,6 +35,6 @@ bash "create resources mapping" do
 					"_all": { "enabled": true } 
 				}
 			}
-		}' http://#{node['Nodes']['ES']}:9200/_template/resources_template
+		}' http://#{es_node}:9200/_template/resources_template
 	EOH
 end
\ No newline at end of file
diff --git a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_3_create_monitoring_template.rb b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_3_create_monitoring_template.rb
index 02b20f8cce..685b468e7a 100644
--- a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_3_create_monitoring_template.rb
+++ b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_3_create_monitoring_template.rb
@@ -1,8 +1,11 @@
+# Get the first es node in the list
+es_node =  node['Nodes']['ES'].first
+
 ruby_block "check_ElasticSearch_Cluster_Health" do
     block do
       #tricky way to load this Chef::Mixin::ShellOut utilities
       Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut)
-      curl_command = "http://#{node['Nodes']['ES']}:9200/_cluster/health?pretty=true"
+      curl_command = "http://#{es_node}:9200/_cluster/health?pretty=true"
       resp = Net::HTTP.get_response URI.parse(curl_command)
       stat = JSON.parse(resp.read_body)['status']
 
@@ -48,6 +51,6 @@ bash "create monitoring mapping" do
 				}
 			},
 			"aliases": { "last_3_months": {} }
-		}' http://#{node['Nodes']['ES']}:9200/_template/monitoring_template
+		}' http://#{es_node}:9200/_template/monitoring_template
 	EOH
 end
diff --git a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_4_create_kibana_dashboard_virtualization.rb b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_4_create_kibana_dashboard_virtualization.rb
index 08c3fac613..53497133f5 100644
--- a/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_4_create_kibana_dashboard_virtualization.rb
+++ b/sdc-os-chef/sdc-init-elasticsearch/chef-repo/cookbooks/init-sdc-elasticsearch/recipes/ES_4_create_kibana_dashboard_virtualization.rb
@@ -1,8 +1,11 @@
+# Get the first es node in the list
+es_node =  node['Nodes']['ES'].first
+
 ruby_block "check_ElasticSearch_Cluster_Health" do
     block do
       #tricky way to load this Chef::Mixin::ShellOut utilities
       Chef::Resource::RubyBlock.send(:include, Chef::Mixin::ShellOut)
-      curl_command = "http://#{node['Nodes']['ES']}:9200/_cluster/health?pretty=true"
+      curl_command = "http://#{es_node}:9200/_cluster/health?pretty=true"
       resp = Net::HTTP.get_response URI.parse(curl_command)
       stat = JSON.parse(resp.read_body)['status']
 
@@ -25,7 +28,7 @@ bash "create Kibana dashboard" do
         for file in /root/chef-solo/cookbooks/sdc-init-elasticsearch/files/default/dashboard_*.json; do 
             name=`basename $file .json | awk -F"_" '{print $2}'` 
             echo "Loading dashboard $name:" 
-            curl -XPUT http://#{node['Nodes']['ES']}:9200/.kibana/dashboard/$name -d @$file || exit 1 
+            curl -XPUT http://#{es_node}:9200/.kibana/dashboard/$name -d @$file || exit 1
             echo 
         done 
     EOH
@@ -36,7 +39,7 @@ bash "create Kibana visualization" do
         for file in /root/chef-solo/cookbooks/sdc-init-elasticsearch/files/default/visualization_*.json; do 
             name=`basename $file .json | awk -F"_" '{print $2}'` 
             echo "Loading visualization $name:" 
-            curl -XPUT http://#{node['Nodes']['ES']}:9200/.kibana/visualization/$name -d @$file || exit 1 
+            curl -XPUT http://#{es_node}:9200/.kibana/visualization/$name -d @$file || exit 1
             echo 
         done 
     EOH