New docker start monitor engine

New docker start monitor engine based on docker log prints

Change-Id: I592b235faa090e558b13520d830bfd9cb6af924b
Issue-Id: SDC-432
Signed-off-by: Yuli Shlosberg <ys9693@att.com>

7 files changed, 90 insertions, 25 deletions

diff --git a/sdc-os-chef/scripts/docker_run.sh b/sdc-os-chef/scripts/docker_run.sh
index 02eff2375f..bc24124651 100755
--- a/sdc-os-chef/scripts/docker_run.sh
+++ b/sdc-os-chef/scripts/docker_run.sh
@@ -22,6 +22,37 @@ function dir_perms {
 	chmod -R 777 /data/logs
 }
 
+function monitor_docker {
+
+echo monitor $1 Docker
+
+TIME_OUT=180
+INTERVAL=20
+TIME=0
+while [ "$TIME" -lt "$TIME_OUT" ]; do
+
+MATCH=`docker logs --tail 30 $1 | grep "DOCKER STARTED"`
+echo MATCH is -- $MATCH
+
+if [ -n "$MATCH" ]
+ then
+    echo DOCKER start finished in $TIME seconds
+    break
+  fi
+
+  echo Sleep: $INTERVAL seconds before testing if $1 DOCKER is up. Total wait time up now is: $TIME seconds. Timeout is: $TIME_OUT seconds
+  sleep $INTERVAL
+  TIME=$(($TIME+$INTERVAL))
+done
+
+if [ "$TIME" -ge "$TIME_OUT" ]
+ then
+   echo -e "\e[1;31mTIME OUT: DOCKER was NOT fully started in $TIME_OUT seconds... Could cause problems ...\e[0m"
+fi
+
+
+}
+
 
 RELEASE=latest
 LOCAL=false
@@ -57,7 +88,7 @@ while [ "$1" != "" ]; do
 			usage
             exit
             ;;
-        * ) 
+        * )
     		usage
             exit 1
     esac
@@ -97,15 +128,16 @@ docker run --detach --name sdc-cs --env RELEASE="${RELEASE}" --env ENVNAME="${DE
 
 
 echo "please wait while CS is starting..."
-echo ""
-c=120 # seconds to wait
-REWRITE="\e[25D\e[1A\e[K"
-while [ $c -gt 0 ]; do
-    c=$((c-1))
-    sleep 1
-    echo -e "${REWRITE}$c"
-done
-echo -e ""
+monitor_docker sdc-cs
+#echo ""
+#c=120 # seconds to wait
+#REWRITE="\e[25D\e[1A\e[K"
+#while [ $c -gt 0 ]; do
+#    c=$((c-1))
+#    sleep 1
+#    echo -e "${REWRITE}$c"
+#done
+#echo -e "
 
 # kibana
 echo "docker run sdc-kibana..."
@@ -124,15 +156,18 @@ fi
 docker run --detach --name sdc-BE --env HOST_IP=${IP} --env ENVNAME="${DEP_ENV}" --env http_proxy=${http_proxy} --env https_proxy=${https_proxy} --env no_proxy=${no_proxy} --log-driver=json-file --log-opt max-size=100m --log-opt max-file=10 --ulimit memlock=-1:-1 --memory 4g --memory-swap=4g --ulimit nofile=4096:100000 --volume /etc/localtime:/etc/localtime:ro --volume /data/logs/BE/:/var/lib/jetty/logs  --volume /data/environments:/root/chef-solo/environments --publish 8443:8443 --publish 8080:8080 ${PREFIX}/sdc-backend:${RELEASE}
 
 echo "please wait while BE is starting..."
-echo ""
-c=45 # seconds to wait
-REWRITE="\e[45D\e[1A\e[K"
-while [ $c -gt 0 ]; do
-    c=$((c-1))
-    sleep 1
-    echo -e "${REWRITE}$c"
-done
-echo -e ""
+monitor_docker sdc-BE
+#echo ""
+#c=45 # seconds to wait
+#REWRITE="\e[45D\e[1A\e[K"
+#while [ $c -gt 0 ]; do
+#    c=$((c-1))
+#    sleep 1
+#    echo -e "${REWRITE}$c"
+#done
+#echo -e ""
+
+
 
 
 # Front-End
@@ -142,6 +177,10 @@ if [ ${LOCAL} = false ]; then
 fi
 docker run --detach --name sdc-FE --env HOST_IP=${IP} --env ENVNAME="${DEP_ENV}" --env http_proxy=${http_proxy} --env https_proxy=${https_proxy} --env no_proxy=${no_proxy} --log-driver=json-file --log-opt max-size=100m --log-opt max-file=10 --ulimit memlock=-1:-1 --memory 2g --memory-swap=2g --ulimit nofile=4096:100000 --volume /etc/localtime:/etc/localtime:ro  --volume /data/logs/FE/:/var/lib/jetty/logs --volume /data/environments:/root/chef-solo/environments --publish 9443:9443 --publish 8181:8181 ${PREFIX}/sdc-frontend:${RELEASE}
 
+echo "docker run sdc-frontend..."
+monitor_docker sdc-FE
+
+
 
 
 # running healthCheck scripts
@@ -170,4 +209,4 @@ echo "Triger sanity docker, please wait..."
 	   docker pull ${PREFIX}/sdc-sanity:${RELEASE}
     fi
 	docker run --detach --name sdc-sanity --env HOST_IP=${IP} --env ENVNAME="${DEP_ENV}" --env http_proxy=${http_proxy} --env https_proxy=${https_proxy} --env no_proxy=${no_proxy} --log-driver=json-file --log-opt max-size=100m --log-opt max-file=10 --ulimit memlock=-1:-1 --memory 512m --memory-swap=512m --ulimit nofile=4096:100000 --volume /etc/localtime:/etc/localtime:ro --volume /data/logs/sdc-sanity/target:/var/lib/tests/target --volume /data/logs/sdc-sanity/ExtentReport:/var/lib/tests/ExtentReport --volume /data/environments:/root/chef-solo/environments --publish 9560:9560 ${PREFIX}/sdc-sanity:${RELEASE}
-fi
+fi
\ No newline at end of file
diff --git a/sdc-os-chef/sdc-backend/startup.sh b/sdc-os-chef/sdc-backend/startup.sh
index a0c237da62..cfdab442d4 100644
--- a/sdc-os-chef/sdc-backend/startup.sh
+++ b/sdc-os-chef/sdc-backend/startup.sh
@@ -39,3 +39,5 @@ fi
 
 while true; do sleep 2; done
 
+echo "###### DOCKER STARTED #####"
+
diff --git a/sdc-os-chef/sdc-cassandra/chef-repo/cookbooks/cassandra-actions/recipes/07-importComformance.rb b/sdc-os-chef/sdc-cassandra/chef-repo/cookbooks/cassandra-actions/recipes/07-importComformance.rb
index f60c6977e0..ce1d50b8ea 100644
--- a/sdc-os-chef/sdc-cassandra/chef-repo/cookbooks/cassandra-actions/recipes/07-importComformance.rb
+++ b/sdc-os-chef/sdc-cassandra/chef-repo/cookbooks/cassandra-actions/recipes/07-importComformance.rb
@@ -18,6 +18,11 @@ bash "import-Comformance" do
     ./sdcSchemaFileImport.sh ${tosca_dir} #{cl_release} ${cl_version} ${conf_dir} 
   EOH
 end
+bash "echo status" do
+   code <<-EOH
+     echo "DOCKER STARTED"
+   EOH
+end
 
 
 
diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb
index dfb68c1dfa..5b1f9f8180 100644
--- a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb
+++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_5_create_monitoring_template.rb
@@ -20,6 +20,12 @@ ruby_block "check_ElasticSearch_Cluster_Health" do
    retry_delay 2
 end
 
+bash "echo status" do
+   code <<-EOH
+     echo "DOCKER STARTED"
+   EOH
+end
+
 bash "create monitoring mapping" do
 	code <<-EOH	
 		curl -i -X PUT -d '{ "order": 1, "template": "monitoring_events-*", "settings": {}, "mappings":
diff --git a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_create_kibana_dashboard_virtualization.rb b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_create_kibana_dashboard_virtualization.rb
index a3b15073a1..216d103fa3 100644
--- a/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_create_kibana_dashboard_virtualization.rb
+++ b/sdc-os-chef/sdc-elasticsearch/chef-repo/cookbooks/sdc-elasticsearch/recipes/ES_6_create_kibana_dashboard_virtualization.rb
@@ -29,6 +29,11 @@ ruby_block "check_ElasticSearch_Cluster_Health" do
    retry_delay 2
 end
 
+bash "echo status" do
+   code <<-EOH
+     echo "DOCKER STARTED"
+   EOH
+end
 
 bash "create Kibana dashboard" do
     code <<-EOH
@@ -41,7 +46,6 @@ bash "create Kibana dashboard" do
     EOH
 end
 
-
 bash "create Kibana visualization" do
     code <<-EOH
         for file in /root/chef-solo/cookbooks/sdc-elasticsearch/files/default/visualization_*.json; do 
@@ -51,7 +55,4 @@ bash "create Kibana visualization" do
             echo 
         done 
     EOH
-end
-
-
-
+end
\ No newline at end of file
diff --git a/sdc-os-chef/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/recipes/FE_7_create_jetty_modules.rb b/sdc-os-chef/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/recipes/FE_7_create_jetty_modules.rb
index fc9dd86f40..5497576f8f 100644
--- a/sdc-os-chef/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/recipes/FE_7_create_jetty_modules.rb
+++ b/sdc-os-chef/sdc-frontend/chef-repo/cookbooks/sdc-catalog-fe/recipes/FE_7_create_jetty_modules.rb
@@ -43,3 +43,9 @@ template "ssl-ini" do
    variables :https_port => "#{node['FE'][:https_port]}"
 end
 
+bash "echo status" do
+   code <<-EOH
+     echo "###### DOCKER STARTED #####"
+   EOH
+end
+
diff --git a/sdc-os-chef/sdc-kibana/chef-repo/cookbooks/sdc-kibana/recipes/setup_kibana.rb b/sdc-os-chef/sdc-kibana/chef-repo/cookbooks/sdc-kibana/recipes/setup_kibana.rb
index ee5d5c74e6..b30893c63f 100644
--- a/sdc-os-chef/sdc-kibana/chef-repo/cookbooks/sdc-kibana/recipes/setup_kibana.rb
+++ b/sdc-os-chef/sdc-kibana/chef-repo/cookbooks/sdc-kibana/recipes/setup_kibana.rb
@@ -17,3 +17,9 @@ template "kibana-yml" do
   })
 end
 
+bash "echo status" do
+   code <<-EOH
+     echo "DOCKER STARTED"
+   EOH
+end
+