Fix config format backwards compatibility

Issue-ID: DCAEGEN2-501
Change-Id: I8638518c7e72547d6fc4262f4e815dede9b9fb78
Signed-off-by: Lusheng Ji <lji@research.att.com>

5 files changed, 121 insertions, 78 deletions

diff --git a/tca-cdap-container/Dockerfile b/tca-cdap-container/Dockerfile
index 158c95a..5cd1267 100644
--- a/tca-cdap-container/Dockerfile
+++ b/tca-cdap-container/Dockerfile
@@ -16,8 +16,7 @@
 
 FROM caskdata/cdap-standalone:4.1.2
 
-RUN apt-get update
-RUN apt-get install -y netcat jq wget vim iputils-ping
+RUN apt-get update && apt-get install -y netcat jq iputils-ping wget vim
 COPY get-tca.sh /opt/tca/get-tca.sh
 RUN /opt/tca/get-tca.sh
 COPY tca_app_config.json /opt/tca/tca_app_config.json
diff --git a/tca-cdap-container/README.txt b/tca-cdap-container/README.txt
deleted file mode 100644
index 62a7a09..0000000
--- a/tca-cdap-container/README.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Note:
-
-Although typically Java jar artifacts have SNAPSHOT version as a.b.c-SNAPSHOT, internally CDAP
-identifies the application as a.b.c.SNAPSHOT.  Thus, in app_config JSON we must refer to the 
-application as a.b.c.SNAPSHOT.  Otherwise we will have artifact not found error"
diff --git a/tca-cdap-container/get-tca.sh b/tca-cdap-container/get-tca.sh
index 66038b1..9b46830 100755
--- a/tca-cdap-container/get-tca.sh
+++ b/tca-cdap-container/get-tca.sh
@@ -1,17 +1,47 @@
 #!/bin/bash
+# ================================================================================
+# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
 
+  
 ARTIFACTPATH=${1:-/opt/tca/}
 PROTO='https'
 NEXUSREPO='nexus.onap.org'
-REPO='snapshots'
 GROUPID='org.onap.dcaegen2.analytics.tca'
 ARTIFACTID='dcae-analytics-cdap-tca'
-VERSION='2.2.0-SNAPSHOT'
 
-URL="${PROTO}://${NEXUSREPO}/service/local/repositories/${REPO}/content/${GROUPID//.//}/${ARTIFACTID}/${VERSION}/maven-metadata.xml"
-VT=$(wget --no-check-certificate -O- $URL | grep -m 1 \<value\> | sed -e 's/<value>\(.*\)<\/value>/\1/' | sed -e 's/ //g')
+#REPO='snapshots'
+REPO='releases'
+VERSION=''
 
+# if VERSION is not specified, find out the latest version
+if [ -z "$VERSION" ]; then
+  URL="${PROTO}://${NEXUSREPO}/service/local/repositories/${REPO}/content/${GROUPID//.//}/${ARTIFACTID}/maven-metadata.xml"
+  VERSION=$(wget --no-check-certificate -O- $URL | grep -m 1 \<latest\> | sed -e 's/<latest>\(.*\)<\/latest>/\1/' | sed -e 's/ //g')
+fi
+
+echo "Getting version $VERSION of $GROUPID.$ARTIFACTID from $REPO repo on $NEXUSREPO"
+
+if [ "$REPO" == "snapshots" ]; then
+  # SNOTSHOT repo container many snapshots for each version.  get the newest among them
+  URL="${PROTO}://${NEXUSREPO}/service/local/repositories/${REPO}/content/${GROUPID//.//}/${ARTIFACTID}/${VERSION}/maven-metadata.xml"
+  VT=$(wget --no-check-certificate -O- $URL | grep -m 1 \<value\> | sed -e 's/<value>\(.*\)<\/value>/\1/' | sed -e 's/ //g')
+else
+  VT=${VERSION}
+fi
 URL="${PROTO}://${NEXUSREPO}/service/local/repositories/${REPO}/content/${GROUPID//.//}/${ARTIFACTID}/${VERSION}/${ARTIFACTID}-${VT}.jar"
-#wget --no-check-certificate "${URL}" -O "${ARTIFACTPATH}${ARTIFACTID}-${VERSION%-SNAPSHOT}.jar"
-wget --no-check-certificate "${URL}" -O "${ARTIFACTPATH}${ARTIFACTID}.${VERSION}.jar"
+echo "Fetching $URL"
 
+wget --no-check-certificate "${URL}" -O "${ARTIFACTPATH}${ARTIFACTID}.${VERSION}.jar"
diff --git a/tca-cdap-container/restart.sh b/tca-cdap-container/restart.sh
index e962ee5..4f6ed92 100755
--- a/tca-cdap-container/restart.sh
+++ b/tca-cdap-container/restart.sh
@@ -23,9 +23,9 @@ TCA_NAMESPACE='cdap_tca_hi_lo'
 TCA_APPNAME='dcae-tca'
 
 TCA_ARTIFACT='dcae-analytics-cdap-tca'
-TCA_ARTIFACT_VERSION='2.2.0-SNAPSHOT'
 TCA_FILE_PATH='/opt/tca'
-TCA_JAR="${TCA_FILE_PATH}/${TCA_ARTIFACT}.${TCA_ARTIFACT_VERSION}.jar"
+TCA_JAR="$(ls -1r ${TCA_FILE_PATH}/${TCA_ARTIFACT}*.jar | head -1)"
+TCA_ARTIFACT_VERSION=$(echo "$TCA_JAR" |rev |cut -f 2-4 -d '.' |rev)
 TCA_APP_CONF="${TCA_FILE_PATH}/tca_app_config.json"
 TCA_CONF="${TCA_FILE_PATH}/tca_config.json"
 TCA_PREF="${TCA_FILE_PATH}/tca_app_preferences.json"
@@ -37,29 +37,15 @@ TCA_PATH_APP="${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/${TCA_NAMESPACE}/apps/${TC
 TCA_PATH_ARTIFACT="${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/${TCA_NAMESPACE}/artifacts"
 
 
-CONSUL_HOST=${CONSU_HOST:-consul}
-CONSUL_PORT=${CONSU_PORT:-8500}
+CONSUL_HOST=${CONSUL_HOST:-consul}
+CONSUL_PORT=${CONSUL_PORT:-8500}
 CONFIG_BINDING_SERVICE=${CONFIG_BINDING_SERVICE:-config_binding_service}
 
 CBS_SERVICE_NAME=${CONFIG_BINDING_SERVICE}
 
-unset CBS_HOST
-unset CBS_PORT
-until [ ! -z "$CBS_HOST" ]; do
-  echo "Retrieving host and port for ${CBS_SERVICE_NAME} from ${CONSUL_HOST}:${CONSUL_PORT}" 
-  sleep 2
-  CBS_HOST=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServiceAddress |sed -e 's/\"//g')
-  CBS_PORT=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServicePort |sed -e 's/\"//g')
-done
-echo "Retrieved host and port for ${CBS_SERVICE_NAME} as ${CBS_HOST}:${CBS_PORT}" 
-CBS_HOST=${CBS_HOST:-config-binding-service}
-CBS_PORT=${CBS_PORT:-10000}
-
 #Changing to HOSTNAME parameter for consistency with k8s deploy
 MY_NAME=${HOSTNAME:-tca}
 
-echo "TCA environment: I am ${MY_NAME}, consul at ${CONSUL_HOST}:${CONSUL_PORT}, CBS at ${CBS_HOST}:${CBS_PORT}, service name ${CBS_SERVICE_NAME}"
-
 
 echo "Generting preference file"
 sed -i 's/{{DMAAPHOST}}/'"${DMAAPHOST}"'/g' ${TCA_PREF}
@@ -70,7 +56,7 @@ sed -i 's/{{DMAAPSUBGROUP}}/OpenDCAEc12/g' ${TCA_PREF}
 sed -i 's/{{DMAAPSUBID}}/c12/g' ${TCA_PREF}
 sed -i 's/{{AAIHOST}}/'"${AAIHOST}"'/g' ${TCA_PREF}
 sed -i 's/{{AAIPORT}}/'"${AAIPORT}"'/g' ${TCA_PREF}
-if [ -z $REDISHOSTPORT ]; then
+if [ -z "$REDISHOSTPORT" ]; then
   sed -i 's/{{REDISHOSTPORT}}/NONE/g' ${TCA_PREF}
   sed -i 's/{{REDISCACHING}}/false/g' ${TCA_PREF}
 else
@@ -149,56 +135,76 @@ function tca_status {
 
 
 function tca_poll_policy {
-    MY_NAME=${HOSTNAME:-tca}
-
     URL0="${CBS_HOST}:${CBS_PORT}/service_component_all/${MY_NAME}"
-    echo "tca_poll_policy: Retrieving configuration file at ${URL0}"
+    echo "tca_poll_policy: Retrieving all-in-one config at ${URL0}"
     HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" "$URL0")
-    HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
-    HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
+    HTTP_BODY=$(echo "$HTTP_RESPONSE" | sed -e 's/HTTPSTATUS\:.*//g')
+    HTTP_STATUS=$(echo "$HTTP_RESPONSE" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
 
     if [ "$HTTP_STATUS" != "200" ]; then
+        echo "tca_poll_policy: Retrieving all-in-one config failed with status $HTTP_STATUS"
         URL1="${CBS_HOST}:${CBS_PORT}/service_component/${MY_NAME}"
-        echo "tca_poll_policy: Retrieving configuration file at ${URL1}"
+        echo "tca_poll_policy: Retrieving app config only at ${URL1}"
         HTTP_RESPONSE1=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" "$URL1")
-        HTTP_BODY1=$(echo $HTTP_RESPONSE1 | sed -e 's/HTTPSTATUS\:.*//g')
-        HTTP_STATUS1=$(echo $HTTP_RESPONSE1 | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
+        HTTP_BODY1=$(echo "$HTTP_RESPONSE1" | sed -e 's/HTTPSTATUS\:.*//g')
+        HTTP_STATUS1=$(echo "$HTTP_RESPONSE1" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
         if [ "$HTTP_STATUS1" != "200" ]; then
-            echo "receiving $HTTP_RESPONSE1 from CBS"
+            echo "tca_poll_policy: Retrieving app config only failed with status $HTTP_STATUS1"
             return
         fi
 
         URL2="$URL1:preferences"
-        echo "tca_poll_policy: Retrieving preferences file at ${URL1}"
+        echo "tca_poll_policy: Retrieving app preferences only at ${URL2}"
         HTTP_RESPONSE2=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" "$URL2")
-        HTTP_BODY2=$(echo $HTTP_RESPONSE2 | sed -e 's/HTTPSTATUS\:.*//g')
-        HTTP_STATUS2=$(echo $HTTP_RESPONSE2 | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
+        HTTP_BODY2=$(echo "$HTTP_RESPONSE2" | sed -e 's/HTTPSTATUS\:.*//g')
+        HTTP_STATUS2=$(echo "$HTTP_RESPONSE2" | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
         if [ "$HTTP_STATUS2" != "200" ]; then
-            echo "receiving $HTTP_RESPONSE2 from CBS"
+            echo "tca_poll_policy: Retrieving app preferences only failed with status $HTTP_STATUS2"
+            return
+        fi
+  
+        if [[ "$CONFIG" == "null"  || "$PREF" == "null" ]]; then
+            echo "tca_poll_policy: either app config or app preferences being empty, config not applicable"
             return
         fi
 
-        echo $HTTP_BODY1 | jq . --sort-keys > "${TCA_CONF_TEMP}"
-        echo $HTTP_BODY2 | jq . --sort-keys > "${TCA_PREF_TEMP}"
+        echo "$HTTP_BODY1" | jq . --sort-keys > "${TCA_CONF_TEMP}"
+        echo "$HTTP_BODY2" | jq . --sort-keys > "${TCA_PREF_TEMP}"
     else
-        CONFIG=$(echo $HTTP_BODY | jq .config.app_config)
-        PREF=$(echo $HTTP_BODY | jq .config.app_preferences)
-        POLICY=$(echo $HTTP_BODY | jq .policies.items[0].config.content.tca_policy)
+        CONFIG=$(echo "$HTTP_BODY" | jq .config.app_config)
+        PREF=$(echo "$HTTP_BODY" | jq .config.app_preferences)
+        POLICY=$(echo "$HTTP_BODY" | jq .policies.items[0].config.content.tca_policy)
 
-	## Check if policy content under tca_policy is returned null
-	## null indicates no active policy flow; hence use configuration loaded 
-	## from blueprint
 
-        if [ $POLICY==null ]; then
-		# tca_policy through blueprint
-		NEWPREF=${PREF}
+        if [[ "$CONFIG" == "null"  || "$PREF" == "null" ]]; then
+            echo "tca_poll_policy: CONFIG received is parsed to be empty, trying to parse using R1 format" 
+            CONFIG=$(echo "$HTTP_BODY" | jq .config)
+            NEWPREF=$(echo "$HTTP_BODY" | jq .preferences)
+
+            #echo "CONFIG is [$CONFIG]"
+            #echo "NEWPREF is [$NEWPREF]"
         else
-		# tca_policy through active policy flow through PH
-        	NEWPREF=$(echo $PREF | jq --arg tca_policy "$POLICY" '. + {$tca_policy}')
+            echo "tca_poll_policy: CONFIG is [${CONFIG}], PREF is [${PREF}], POLICY is [${POLICY}]"
+	    ## Check if policy content under tca_policy is returned null
+	    ## null indicates no active policy flow; hence use configuration loaded 
+	    ## from blueprint
+            if [ "$POLICY" == "null" ]; then
+                # tca_policy through blueprint
+                NEWPREF=${PREF}
+            else
+                # tca_policy through active policy flow through PH
+                NEWPREF=$(echo "$PREF" | jq --arg tca_policy "$POLICY" '. + {$tca_policy}')
+            fi
+            NEWPREF=$(echo "$NEWPREF" | sed 's/\\n//g') 
+        fi
+       
+        if [[ "$CONFIG" == "null"  || "$NEWPREF" == "null" ]]; then
+             echo "tca_poll_policy: either app config or app preferences being empty, config not applicable"
+             return
         fi
-        NEWPREF=$(echo $NEWPREF | sed 's/\\n//g')
-        echo $CONFIG | jq . --sort-keys > "${TCA_CONF_TEMP}"
-        echo $NEWPREF | jq . --sort-keys > "${TCA_PREF_TEMP}"
+
+        echo "$CONFIG" | jq . --sort-keys > "${TCA_CONF_TEMP}"
+        echo "$NEWPREF" | jq . --sort-keys > "${TCA_PREF_TEMP}"
     fi
 
     if [ ! -e "${TCA_CONF_TEMP}" ] || [ "$(ls -sh ${TCA_CONF_TEMP} |cut -f1 -d' ' |sed -e 's/[^0-9]//g')"  -lt "1" ]; then
@@ -229,7 +235,7 @@ function tca_poll_policy {
     CONSUMERID=$(jq .subscriberConsumerId ${TCA_PREF_TEMP} |sed -e 's/\"//g')
     if ! (echo "$CONSUMERID" |grep "$HOSTID"); then
         CONSUMERID="${CONSUMERID}-${HOSTID}"
-        jq --arg CID ${CONSUMERID} '.subscriberConsumerId = $CID' < "${TCA_PREF_TEMP}" > "${TCA_PREF_TEMP}2"
+        jq --arg CID "${CONSUMERID}" '.subscriberConsumerId = $CID' < "${TCA_PREF_TEMP}" > "${TCA_PREF_TEMP}2"
         mv "${TCA_PREF_TEMP}2" "${TCA_PREF_TEMP}"
     fi 
     if ! diff ${TCA_PREF} ${TCA_PREF_TEMP} ; then
@@ -238,7 +244,8 @@ function tca_poll_policy {
         PERF_CHANGED=1
     fi
 
-    if [[ "$PERF_CHANGED" == "1" || "$CONF_CHANGED" == "1" ]]; then 
+    if [[ "$PERF_CHANGED" == "1" || "$CONF_CHANGED" == "1" ]]; then
+        echo "Newly received configuration/preference differ from the running instance's.  reload confg"
 	tca_stop
 	tca_delete
         tca_load_artifact
@@ -251,48 +258,60 @@ function tca_poll_policy {
 
 export PATH=${PATH}:/opt/cdap/sdk/bin
 
-# starting CDAP SDK in background
-cdap sdk start 
 
+echo "Starting TCA-CDAP in standalone mode"
 
+# starting CDAP SDK in background
+cdap sdk start 
 
-echo "Waiting CDAP ready on port 11015 ..."
+echo "Started, waiting CDAP ready on port 11015 ..."
 while ! nc -z ${CDAP_HOST} ${CDAP_PORT}; do   
   sleep 0.1 # wait for 1/10 of the second before check again
 done
-echo "CDAP has started"
-
 
 echo "Creating namespace cdap_tca_hi_lo ..."
 curl -s -X PUT "http://${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/cdap_tca_hi_lo"
 
-
 # stop programs
 tca_stop
 
-
 # delete application
 tca_delete
 
-
 # load artifact
 tca_load_artifact
 tca_load_conf
 
-
 # start programs
 tca_start
 
-
 # get status of programs
 tca_status
 
+echo "TCA-CDAP standalone mode initialization completed"
+
 
 
-while echo -n
+#Changing to HOSTNAME parameter for consistency with k8s deploy
+MY_NAME=${HOSTNAME:-tca}
+
+unset CBS_HOST
+unset CBS_PORT
+echo "TCA environment: I am ${MY_NAME}, consul at ${CONSUL_HOST}:${CONSUL_PORT}, CBS service name ${CBS_SERVICE_NAME}"
+
+while echo
 do
-    echo "======================================================"
-    date
-    tca_poll_policy
+    echo "$(date):  ======================================================"
+    if [[ -z "$CBS_HOST" ||  -z "$CBS_PORT" ]]; then
+       echo "Retrieving host and port for ${CBS_SERVICE_NAME} from ${CONSUL_HOST}:${CONSUL_PORT}"
+       sleep 2
+       CBS_HOST=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServiceAddress |sed -e 's/\"//g')
+       CBS_PORT=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServicePort |sed -e 's/\"//g')
+       echo "CBS discovered to be at ${CBS_HOST}:${CBS_PORT}"
+    fi
+
+    if [ ! -z "$CBS_HOST" ] && [ ! -z "$CBS_PORT" ]; then
+       tca_poll_policy
+    fi
     sleep 30
 done
diff --git a/tca-cdap-container/tca_app_config.json b/tca-cdap-container/tca_app_config.json
index d6adcb7..24234dc 100644
--- a/tca-cdap-container/tca_app_config.json
+++ b/tca-cdap-container/tca_app_config.json
@@ -2,7 +2,7 @@
   "artifact": {
     "name": "dcae-analytics-cdap-tca",
     "scope": "user",
-    "version": "2.2.0.SNAPSHOT"
+    "version": "2.2.1"
   },
   "config": {
     "appDescription": "DCAE Analytics Threshold Crossing Alert Application",