From d6d409f7dbb14e4df2f326684c07db3a8859d531 Mon Sep 17 00:00:00 2001 From: Lusheng Ji Date: Thu, 22 Mar 2018 23:25:45 -0400 Subject: Add TCA-CDAP container Issue-ID: DCAEGEN2-357 Change-Id: I52a4f1c684aee297d248b1e7df5802d9439930ec Signed-off-by: Lusheng Ji --- mvn-phase-script.sh | 2 +- pom.xml | 1 + tca-cdap-container/Dockerfile | 35 +++++ tca-cdap-container/host.aliases | 1 + tca-cdap-container/pom.xml | 172 +++++++++++++++++++++ tca-cdap-container/restart.sh | 231 ++++++++++++++++++++++++++++ tca-cdap-container/tca_app_config.json | 19 +++ tca-cdap-container/tca_app_preferences.json | 33 ++++ 8 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 tca-cdap-container/Dockerfile create mode 100644 tca-cdap-container/host.aliases create mode 100644 tca-cdap-container/pom.xml create mode 100644 tca-cdap-container/restart.sh create mode 100644 tca-cdap-container/tca_app_config.json create mode 100644 tca-cdap-container/tca_app_preferences.json diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh index 5b1bf07..26d4671 100755 --- a/mvn-phase-script.sh +++ b/mvn-phase-script.sh @@ -91,7 +91,7 @@ deploy) upload_files_of_extension sh build_and_push_docker ;; - k8s-bootstrap|tca-cdap-container|cm-container|redis-cluster-container) + k8s-bootstrap|tca-cdap-container|cmcontainer|redis-cluster-container) build_and_push_docker ;; scripts|cloud_init) diff --git a/pom.xml b/pom.xml index a197237..d9ab7aa 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ limitations under the License. scripts cloud_init redis-cluster-container + tca-cdap-container diff --git a/tca-cdap-container/Dockerfile b/tca-cdap-container/Dockerfile new file mode 100644 index 0000000..f5e70c8 --- /dev/null +++ b/tca-cdap-container/Dockerfile @@ -0,0 +1,35 @@ +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +FROM caskdata/cdap-standalone:4.1.2 + +RUN apt-get update +RUN apt-get install -y netcat jq +ADD https://nexus.onap.org/content/repositories/snapshots/org/onap/dcaegen2/analytics/tca/dcae-analytics-cdap-tca/2.2.0-SNAPSHOT/dcae-analytics-cdap-tca-2.2.0-20180320.190629-1.jar /opt/tca/dcae-analytics-cdap-tca-2.2.0.jar +COPY tca_app_config.json /opt/tca/tca_app_config.json +COPY tca_app_preferences.json /opt/tca/tca_app_preferences.json +COPY restart.sh /opt/tca/restart.sh +COPY host.aliases /etc/host.aliases + +RUN echo "export HOSTALIASES=/etc/host.aliases" >> /etc/profile +RUN chmod 755 /opt/tca/restart.sh + +ENV HOSTALIASES /etc/host.aliases + +EXPOSE 11011 +EXPOSE 11015 + +ENTRYPOINT /opt/tca/restart.sh diff --git a/tca-cdap-container/host.aliases b/tca-cdap-container/host.aliases new file mode 100644 index 0000000..6b0b785 --- /dev/null +++ b/tca-cdap-container/host.aliases @@ -0,0 +1 @@ +consul consul-server.onap-consul.svc.cluster.local diff --git a/tca-cdap-container/pom.xml b/tca-cdap-container/pom.xml new file mode 100644 index 0000000..cab867b --- /dev/null +++ b/tca-cdap-container/pom.xml @@ -0,0 +1,172 @@ + + + + 4.0.0 + + org.onap.dcaegen2.deployments + deployments + 1.2.0-SNAPSHOT + + org.onap.dcaegen2.deployments + tca-cdap-container + dcaegen2-deployments-tca-cdap-container + 1.0.0 + http://maven.apache.org + + UTF-8 + true + . + + + + + py + Python + **/*.py + + + + + ${project.artifactId}-${project.version} + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + clean phase script + clean + + exec + + + + ${project.artifactId} + clean + + + + + generate-sources script + generate-sources + + exec + + + + ${project.artifactId} + generate-sources + + + + + compile script + compile + + exec + + + + ${project.artifactId} + compile + + + + + package script + package + + exec + + + + ${project.artifactId} + package + + + + + test script + test + + exec + + + + ${project.artifactId} + test + + + + + install script + install + + exec + + + + ${project.artifactId} + install + + + + + deploy script + deploy + + exec + + + + ${project.artifactId} + deploy + + + + + + + + diff --git a/tca-cdap-container/restart.sh b/tca-cdap-container/restart.sh new file mode 100644 index 0000000..c9c1d56 --- /dev/null +++ b/tca-cdap-container/restart.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +set -e + +CDAP_HOST='localhost' +CDAP_PORT='11015' +TCA_NAMESPACE='cdap_tca_hi_lo' +TCA_APPNAME='dcae-tca' + +TCA_ARTIFACT='dcae-analytics-cdap-tca' +TCA_ARTIFACT_VERSION='2.2.0' +TCA_FILE_PATH='/opt/tca' +TCA_JAR="${TCA_FILE_PATH}/${TCA_ARTIFACT}-${TCA_ARTIFACT_VERSION}.jar" +TCA_APP_CONF="${TCA_FILE_PATH}/tca_app_config.json" +TCA_CONF="${TCA_FILE_PATH}/tca_config.json" +TCA_PREF="${TCA_FILE_PATH}/tca_app_preferences.json" +TCA_CONF_TEMP='/tmp/tca_config.json' +TCA_APP_CONF_TEMP='/tmp/tca_app_config.json' +TCA_PREF_TEMP='/tmp/tca_preferences.json' + +TCA_PATH_APP="${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/${TCA_NAMESPACE}/apps/${TCA_APPNAME}" +TCA_PATH_ARTIFACT="${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/${TCA_NAMESPACE}/artifacts" + +if [ -z "$CONSUL_HOST" ]; then CONSUL_HOST='consul'; fi +if [ -z "$CONSUL_PORT" ]; then CONSUL_PORT='8500'; fi +if [ -z "$CONFIG_BINDING_SERVICE" ]; then CONFIG_BINDING_SERVICE='config_binding_service'; fi + + +function tca_stop { + # stop programs + echo + echo "Stopping TCADMaaPMRPublisherWorker, TCADMaaPMRSubscriberWorker, and TCAVESCollectorFlow ..." + echo + curl -s -X POST "http://${TCA_PATH_APP}/workers/TCADMaaPMRPublisherWorker/stop" + curl -s -X POST "http://${TCA_PATH_APP}/workers/TCADMaaPMRSubscriberWorker/stop" + curl -s -X POST "http://${TCA_PATH_APP}/flows/TCAVESCollectorFlow/stop" + echo "done" + echo +} + +function tca_load_artifact { + echo + echo "Loading artifact ..." + curl -s -X POST --data-binary @"${TCA_JAR}" "http://${TCA_PATH_ARTIFACT}/${TCA_ARTIFACT}" + echo +} + +function tca_load_conf { + echo + echo "Loading configuration ..." + curl -s -X PUT -d @${TCA_APP_CONF} http://${TCA_PATH_APP} + echo + + # load preferences + echo + echo "Loading preferences ..." + curl -s -X PUT -d @${TCA_PREF} http://${TCA_PATH_APP}/preferences + echo +} + + +function tca_delete { + echo + echo "Deleting application dcae-tca ..." + curl -s -X DELETE http://${TCA_PATH_APP} + echo + + # delete artifact + echo + echo "Deleting artifact dcae-analytics-cdap-tca version ${TCA_ARTIFACT_VERSION} ..." + curl -s -X DELETE "http://${TCA_PATH_ARTIFACT}/${TCA_ARTIFACT}/versions/${TCA_ARTIFACT_VERSION}" + echo +} + +function tca_start { + echo + echo "Starting TCADMaaPMRPublisherWorker, TCADMaaPMRSubscriberWorker, and TCAVESCollectorFlow ..." + curl -s -X POST "http://${TCA_PATH_APP}/workers/TCADMaaPMRPublisherWorker/start" + curl -s -X POST "http://${TCA_PATH_APP}/workers/TCADMaaPMRSubscriberWorker/start" + curl -s -X POST "http://${TCA_PATH_APP}/flows/TCAVESCollectorFlow/start" + echo +} + + +function tca_status { + echo + echo "TCADMaaPMRPublisherWorker status: " + curl -s "http://${TCA_PATH_APP}/workers/TCADMaaPMRPublisherWorker/status" + echo + echo "TCADMaaPMRSubscriberWorker status: " + curl -s "http://${TCA_PATH_APP}/workers/TCADMaaPMRSubscriberWorker/status" + echo + echo "TCAVESCollectorFlow status" + curl -s "http://${TCA_PATH_APP}/flows/TCAVESCollectorFlow/status" + echo; echo +} + + +function tca_poll_policy { + #CBS_HOST=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServiceAddress |sed -e 's/\"//g') + #CBS_PORT=$(curl -s "${CONSUL_HOST}:${CONSUL_PORT}/v1/catalog/service/${CBS_SERVICE_NAME}" |jq .[0].ServicePort |sed -e 's/\"//g') + + CBS_HOST='bd-service-dcaegen2-platform-cbs.default' + CBS_PORT='10000' + if [ -z "${CBS_HOST}" ] || [ -z "${CBS_PORT}" ]; then + echo 'CBS HOST or PORT not defined' + return + fi + + + MY_NAME=${SERVICE_NAME:-tca} + #MY_NAME=$(hostname |rev |cut -f 3- -d '-' |rev) + + URL1="${CBS_HOST}:${CBS_PORT}/service_component/${MY_NAME}" + URL2="$URL1:preferences" + + echo "Retrieving configuration file at ${URL1}" + curl -s "$URL1" | jq . --sort-keys > "${TCA_CONF_TEMP}" + echo "Retrieving preferences file at ${URL1}" + curl -s "$URL2" | jq . --sort-keys > "${TCA_PREF_TEMP}" + + if [ ! -e "${TCA_CONF_TEMP}" ] || [ "$(ls -sh ${TCA_CONF_TEMP} |cut -f1 -d' ' |sed -e 's/[^0-9]//g')" -lt "1" ]; then + echo "Fail to receive configuration" + return + fi + if [ ! -e "${TCA_PREF_TEMP}" ] || [ "$(ls -sh ${TCA_PREF_TEMP} |cut -f1 -d' ' |sed -e 's/[^0-9]//g')" -lt "1" ]; then + echo "Fail to receive preferences" + return + fi + + CONF_CHANGED="" + # extract only the config section from APP CONF (which has both artifact and config sections) + jq .config --sort-keys ${TCA_APP_CONF} > ${TCA_CONF} + if ! diff ${TCA_CONF} ${TCA_CONF_TEMP} ; then + echo "TCA config changed" + # generating the new app conf using current app conf's artifact section and the new downloaded config + jq --argfile CONFVALUE ${TCA_CONF_TEMP} '.config = $CONFVALUE' <${TCA_APP_CONF} > ${TCA_APP_CONF_TEMP} + + mv ${TCA_APP_CONF_TEMP} ${TCA_APP_CONF} + CONF_CHANGED=1 + fi + + PERF_CHANGED="" + # update the subscriber ConsumerID, if not already unique, + # so replicas appear as different consumers in the consumer group + HOSTID=$(head -1 /etc/hostname | rev |cut -f1-2 -d'-' |rev) + CONSUMERID=$(jq .subscriberConsumerId ${TCA_PREF_TEMP} |sed -e 's/\"//g') + if ! (echo "$CONSUMERID" |grep "$HOSTID"); then + CONSUMERID="${CONSUMERID}-${HOSTID}" + jq --arg CID ${CONSUMERID} '.subscriberConsumerId = $CID' < "${TCA_PREF_TEMP}" > "${TCA_PREF_TEMP}2" + mv "${TCA_PREF_TEMP}2" "${TCA_PREF_TEMP}" + fi + if ! diff ${TCA_PREF} ${TCA_PREF_TEMP} ; then + echo "TCA preference updated" + mv ${TCA_PREF_TEMP} ${TCA_PREF} + PERF_CHANGED=1 + fi + + if [[ "$PERF_CHANGED" == "1" || "$CONF_CHANGED" == "1" ]]; then + tca_stop + tca_delete + tca_load_artifact + tca_load_conf + tca_start + tca_status + fi +} + + +export PATH=${PATH}:/opt/cdap/sdk/bin + +# starting CDAP SDK in background +cdap sdk start + + + +echo "Waiting CDAP ready on port 11015 ..." +while ! nc -z ${CDAP_HOST} ${CDAP_PORT}; do + sleep 0.1 # wait for 1/10 of the second before check again +done +echo "CDAP has started" + + +echo "Creating namespace cdap_tca_hi_lo ..." +curl -s -X PUT "http://${CDAP_HOST}:${CDAP_PORT}/v3/namespaces/cdap_tca_hi_lo" + + +# stop programs +tca_stop + + +# delete application +tca_delete + + +# load artifact +tca_load_artifact +tca_load_conf + + +# start programs +tca_start + + +# get status of programs +tca_status + + + +while echo -n +do + echo "======================================================" + date + tca_poll_policy + sleep 30 +done diff --git a/tca-cdap-container/tca_app_config.json b/tca-cdap-container/tca_app_config.json new file mode 100644 index 0000000..b3a66cc --- /dev/null +++ b/tca-cdap-container/tca_app_config.json @@ -0,0 +1,19 @@ +{ + "artifact": { + "name": "dcae-analytics-cdap-tca", + "scope": "user", + "version": "2.2.0" + }, + "config": { + "appDescription": "DCAE Analytics Threshold Crossing Alert Application", + "appName": "dcae-tca", + "tcaAlertsAbatementTableName": "TCAAlertsAbatementTable", + "tcaAlertsAbatementTableTTLSeconds": 1728000, + "tcaSubscriberOutputStreamName": "TCASubscriberOutputStream", + "tcaVESAlertsTableName": "TCAVESAlertsTable", + "tcaVESAlertsTableTTLSeconds": 1728000, + "tcaVESMessageStatusTableName": "TCAVESMessageStatusTable", + "tcaVESMessageStatusTableTTLSeconds": 86400, + "thresholdCalculatorFlowletInstances": 2 + } +} diff --git a/tca-cdap-container/tca_app_preferences.json b/tca-cdap-container/tca_app_preferences.json new file mode 100644 index 0000000..4ca1a44 --- /dev/null +++ b/tca-cdap-container/tca_app_preferences.json @@ -0,0 +1,33 @@ +{ + "aaiEnrichmentHost": "10.0.1.1", + "aaiEnrichmentIgnoreSSLCertificateErrors": "true", + "aaiEnrichmentPortNumber": "8443", + "aaiEnrichmentProtocol": "https", + "aaiEnrichmentUserName": "DCAE", + "aaiEnrichmentUserPassword": "DCAE", + "aaiVMEnrichmentAPIPath": "/aai/v11/search/nodes-query", + "aaiVNFEnrichmentAPIPath": "/aai/v11/network/generic-vnfs/generic-vnf", + "enableAAIEnrichment": "false", + "enableAlertCEFFormat": "false", + "enableRedisCaching": "false", + "redisHosts": "redis-cluster:6379", + "publisherContentType": "application/json", + "publisherHostName": "dmaap.onap-message-router", + "publisherHostPort": "3904", + "publisherMaxBatchSize": "1", + "publisherMaxRecoveryQueueSize": "100000", + "publisherPollingInterval": "20000", + "publisherProtocol": "http", + "publisherTopicName": "unauthenticated.TCA_EVENT_OUTPUT", + "subscriberConsumerGroup": "OpenDCAE-c12", + "subscriberConsumerId": "c12", + "subscriberContentType": "application/json", + "subscriberHostName": "dmaap.onap-message-router", + "subscriberHostPort": "3904", + "subscriberMessageLimit": "-1", + "subscriberPollingInterval": "30000", + "subscriberProtocol": "http", + "subscriberTimeoutMS": "-1", + "subscriberTopicName": "unauthenticated.SEC_MEASUREMENT_OUTPUT", + "tca_policy": "{\"domain\":\"measurementsForVfScaling\",\"metricsPerEventName\":[{\"eventName\":\"vFirewallBroadcastPackets\",\"controlLoopSchemaType\":\"VNF\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":300,\"direction\":\"LESS_OR_EQUAL\",\"severity\":\"MAJOR\",\"closedLoopEventStatus\":\"ONSET\"},{\"closedLoopControlName\":\"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":700,\"direction\":\"GREATER_OR_EQUAL\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]},{\"eventName\":\"vLoadBalancer\",\"controlLoopSchemaType\":\"VM\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vDNS-6f37f56d-a87d-4b85-b6a9-cc953cf779b3\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":300,\"direction\":\"GREATER_OR_EQUAL\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]},{\"eventName\":\"Measurement_vGMUX\",\"controlLoopSchemaType\":\"VNF\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value\",\"thresholdValue\":0,\"direction\":\"EQUAL\",\"severity\":\"MAJOR\",\"closedLoopEventStatus\":\"ABATED\"},{\"closedLoopControlName\":\"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value\",\"thresholdValue\":0,\"direction\":\"GREATER\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]}]}" +} -- cgit 1.2.3-korg