Add health and service endpoints

4.0.0 deployment-handler
Upgraded cloudify api from 2.1 to 3.1 due to new cloudify /status api
Requires upgrading Cloudify to 4.x
Added 2 APIs to check health of DH itself and to Inventory and Cloudify
Coverage summary
Statements   : 80.75% ( 994/1231 )
Branches     : 57.74% ( 302/523 )
Functions    : 81.13% ( 172/212 )
Lines        : 81.26% ( 984/1211 )

Issue-ID: DCAEGEN2-1127

Change-Id: Ic851a576eed668016f6cfe956f0f1880a65e1065
Signed-off-by: Shadi Haidar <sh1986@att.com>

13 files changed, 556 insertions, 43 deletions

diff --git a/deployment-handler-API.yaml b/deployment-handler-API.yaml
index b910f9b..9359d24 100644
--- a/deployment-handler-API.yaml
+++ b/deployment-handler-API.yaml
@@ -1,5 +1,5 @@
 # ================================================================================
-# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
 swagger: '2.0'
 
 info:
-  version: "5.0.0"
+  version: "5.1.0"
   title: "deployment-handler API"
   license:
     name: "Apache 2.0"
@@ -375,6 +375,82 @@ paths:
         200:
           description: deployment-handler found components with or without policies in cloudify
 
+  /healthcheck:
+    get:
+      tags:
+        - "healthcheck"
+      description: Returns version information and links to API operations thus checking internal health of deployment handler
+      produces:
+        - "application/json"
+      responses:
+
+        200:
+          description: Success
+          schema:
+            title: DispatcherInfo
+            type: object
+            properties:
+              apiVersion:
+                type: string
+                description: |
+                  version of API supported by this server
+              serverVersion:
+                type: string
+                description: |
+                  version of software running on this server
+              links:
+                type: object
+                description: |
+                  Links to API resources
+                properties:
+                  info:
+                    type: string
+                    description: |
+                      path for the server information endpoint
+                  events:
+                    type: string
+                    description: |
+                      path for the events endpoint
+
+  /servicehealth:
+    get:
+      tags:
+        - "servicehealth"
+      description: checks deployment handler's dependencies/external interfaces' health; namely inventory and cloudify
+      produces:
+        - "application/json"
+      responses:
+
+        200:
+          description: Success
+          schema:
+            title: DeploymentHanlderServiceHealth
+            type: object
+            properties:
+              requestId:
+                type: string
+                description: |
+                  Internal request id (for tracking purposes)
+              status:
+                type: string
+                description: |
+                  Status of the API call: OK or NOT OK
+
+        500:
+          description: |
+            Problem on the server side.  See the message in the response for more details.
+          schema:
+            $ref: "#/definitions/DCAEErrorResponse"
+
+        502:
+          description: |
+            Error reported to the dispatcher by a downstream system.  See the message in the response for more details.
+          schema:
+            $ref: "#/definitions/DCAEErrorResponse"
+
+        503:
+          description: |
+            Error communicating with a downstream system(s). Inventory/Cloudify service is not available.
 
 definitions:
 
diff --git a/deployment-handler.js b/deployment-handler.js
index 26074e7..1132af4 100644
--- a/deployment-handler.js
+++ b/deployment-handler.js
@@ -18,7 +18,7 @@ See the License for the specific language governing permissions and limitations
 
 "use strict";
 
-const API_VERSION = "5.0.0";
+const API_VERSION = "5.1.0";
 
 const http = require('http');
 const https = require('https');
@@ -32,6 +32,8 @@ const INFO_PATH = "/";
 const DEPLOYMENTS_PATH = "/dcae-deployments";
 const POLICY_PATH = "/policy";
 const SWAGGER_UI_PATH = "/swagger-ui";
+const HEALTH_CHECK = "/healthcheck";
+const SERVICE_HEALTH = "/servicehealth";
 
 const app = express();
 
@@ -47,10 +49,11 @@ const set_app = function() {
 	app.use(require('./lib/auth').checkAuth);
 
 	/* Set up API routes */
-	app.use(INFO_PATH, require('./lib/info'));
+	app.use([HEALTH_CHECK, INFO_PATH], require('./lib/info'));
 	app.use(DEPLOYMENTS_PATH, require('./lib/dcae-deployments'));
 	app.use(POLICY_PATH, require('./lib/policy'));
 	app.use(SWAGGER_UI_PATH, require('./lib/swagger-ui'));
+	app.use(SERVICE_HEALTH, require('./lib/service-health'));
 
 	/* Set up error handling */
 	app.use(require('./lib/middleware').handleErrors);
@@ -70,6 +73,8 @@ const start = function(config) {
 	config.apiVersion = API_VERSION;
 	config.apiLinks = {
 		"info" : INFO_PATH,
+		"internal-health" : HEALTH_CHECK,
+		"service-health" : SERVICE_HEALTH,
 		"deployments": DEPLOYMENTS_PATH,
 		"policy": POLICY_PATH,
 		"swagger-ui": SWAGGER_UI_PATH
@@ -79,6 +84,13 @@ const start = function(config) {
 	log.info(null, "Configuration: " + JSON.stringify(config, utils.hideSecrets));
 	console.log((new Date()) + ": Configuration: " + JSON.stringify(config, utils.hideSecrets, 2) );
 
+	var cfy = require("./lib/cloudify.js");
+	/* Set config for interface library */
+	cfy.setAPIAddress(config.cloudify.url);
+	cfy.setCredentials(config.cloudify.user, config.cloudify.password);
+	cfy.setLogger(log);
+	process.mainModule.exports.cfy = cfy;
+
 	set_app();
 
 	/* Start the server */
diff --git a/lib/cloudify.js b/lib/cloudify.js
index 1cd5489..49c26dd 100644
--- a/lib/cloudify.js
+++ b/lib/cloudify.js
@@ -318,6 +318,16 @@ exports.getOutputDescriptions = function(req, dpid) {
     return doRequest(req, reqOptions, null, CLOUDIFY);
 };
 
+exports.getCfyStatus = function(req) {
+    const reqOptions = {
+        method : "GET",
+        uri : cfyAPI + "/status"
+    };
+    addAuthToOptions(reqOptions, req);
+
+    return doRequest(req, reqOptions, null, CLOUDIFY);
+};
+
 // Deletes a deployment
 exports.deleteDeployment = function(req, dpid) {
     const reqOptions = {
diff --git a/lib/config.js b/lib/config.js
index d4fd3e3..6430e36 100644
--- a/lib/config.js
+++ b/lib/config.js
@@ -1,5 +1,5 @@
 /*
-Copyright(c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+Copyright(c) 2017-2019 AT&T Intellectual Property. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -79,7 +79,7 @@ const CONFIG_KEY = "deployment_handler";	/* Configuration is stored under the na
 const CM_NAME = "cloudify_manager";
 const INV_NAME = "inventory";
 
-const CM_API_PATH = "/api/v2.1";
+const CM_API_PATH = "/api/v3.1";
 const INV_API_PATH = "";
 
 const DEFAULT_CLOUDIFY_PROTOCOL = "https";
@@ -227,7 +227,6 @@ exports.configure = function() {
 		const missing = findMissingConfig(config);
 		if (missing.length > 0) {
 			throw new Error ("Required configuration elements missing: " + missing.join(','));
-			config = null;
 		}
 		console.log((new Date()) + ": config -> " + JSON.stringify(config, utils.hideSecrets, 2));
 		return config;
diff --git a/lib/deploy.js b/lib/deploy.js
index 4829040..cb78f6f 100644
--- a/lib/deploy.js
+++ b/lib/deploy.js
@@ -31,12 +31,7 @@ const createError = require('./dispatcher-error').createDispatcherError;
 var logger = require("./logging").getLogger();
 
 /* Set up the Cloudify low-level interface library */
-var cfy = require("./cloudify.js");
-/* Set config for interface library */
-cfy.setAPIAddress(config.cloudify.url);
-cfy.setCredentials(config.cloudify.user, config.cloudify.password);
-cfy.setLogger(logger);
-
+const cfy = process.mainModule.exports.cfy;
 
 
 
diff --git a/lib/inventory.js b/lib/inventory.js
index 5935067..0d06856 100644
--- a/lib/inventory.js
+++ b/lib/inventory.js
@@ -24,6 +24,7 @@ const createError = require('./dispatcher-error').createDispatcherError;
 
 const INV_SERV_TYPES = '/dcae-service-types';
 const INV_SERVICES = '/dcae-services';
+const INV_SERVICE_HEALTH = '/servicehealth';
 
 /*
  * Common error handling for inventory API calls
@@ -162,6 +163,29 @@ module.exports = function(options) {
                     return invError(err);
                 }
             });
+        },
+
+        /*
+        * Check if inventory service is healthy using inventory's service check api
+        */
+
+        isServiceHealthy: function(req) {
+            return doRequest(req, {
+                method: "GET",
+                uri: url + INV_SERVICE_HEALTH
+            }, null, INVENTORY)
+
+            .then(function(res) {
+                if ( res.status == 200 ) {
+                    return true;
+                }
+                else {
+                   return false;
+                }
+            },
+            function (err) {
+                return invError(err);
+            });
         }
     };
 };
diff --git a/lib/service-health.js b/lib/service-health.js
new file mode 100644
index 0000000..0b36b88
--- /dev/null
+++ b/lib/service-health.js
@@ -0,0 +1,120 @@
+/*
+Copyright(c) 2019 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/* Handle the /servicehealth API which checks if inventory and cloudify are healthy*/
+
+"use strict";
+
+const app = require('express')();
+app.set('x-powered-by', false);
+app.set('etag', false);
+
+const bodyParser = require('body-parser');
+const inv = require('./inventory');
+const log = require('./logging').getLogger();
+
+/* Pick up config and cloudify exported by main */
+const config = process.mainModule.exports.config;
+const cfy = process.mainModule.exports.cfy;
+const inventory = inv({url: config.inventory.url});
+
+app.use(bodyParser.json({strict: true}));				// Parse body as JSON
+app.use(function (req, res, next) {
+    log.info(req.dcaeReqId,
+        "new req: " + req.method + " " + req.originalUrl +
+        " from: " + req.ip + " body: " + JSON.stringify(req.body)
+    );
+    next();
+});
+
+
+/* Accept an incoming service health check request */
+app.get('/', function (req, res, next) {
+
+    /* Verify inventory service health*/
+    inventory.isServiceHealthy(req)
+
+    .then(function (isInvHealthy) {
+        log.info(req.dcaeReqId,"Checking isInventoryHealthy: " + isInvHealthy);
+        if ( isInvHealthy === true) {
+            log.info(req.dcaeReqId,"Inventory is healthy: ");
+            return cfy.getCfyStatus(req)
+
+                .then(function (cfyStatusResp) {
+                    log.info(req.dcaeReqId,"getCfyStatus Response -> " + JSON.stringify(cfyStatusResp, 2));
+                    if (cfyStatusResp.status === 200) {
+                        var isCfyHealthy = true;
+                        if ( cfyStatusResp.json && cfyStatusResp.json.status === "running" ) {
+                            log.info(req.dcaeReqId,"getCfyStatus Response status is running: ");
+                            const services = Object.keys(cfyStatusResp.json.services);
+                            log.info(req.dcaeReqId,"getCfyStatus services.length -> " + services.length);
+                            // run through all the cfy services to see which ones are running
+                            // don't stop looping when a cfy service is down, it's better to see a list of all serivces' status
+                            // so that we an idea about overall health of cfy
+                            services.forEach( service => {
+                                const service_display_name = cfyStatusResp.json.services[service].display_name;
+                                const instances = Object.keys(cfyStatusResp.json.services[service].instances);
+                                instances.forEach( instance => {
+                                    var description = cfyStatusResp.json.services[service].instances[instance].Description;
+                                    var subState = cfyStatusResp.json.services[service].instances[instance].SubState;
+                                    log.info(req.dcaeReqId,"cfy status for service display_name: " + service_display_name
+                                        + " and Description: " + description + " has a SubState of: " + subState);
+                                    if ( subState !== "running" ) {
+                                        log.info(req.dcaeReqId,"getCfyStatus Description-> " + description + " NOT running ok");
+                                        isCfyHealthy = false;
+                                        res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                                    }
+                                });
+                            });
+                        }
+                        if ( isCfyHealthy === true ) {
+                            log.info(req.dcaeReqId,"Cloudify is healthy");
+                            res.status(200).json({requestId: req.dcaeReqId, status: 'OK'});
+                        }
+                        else {
+                            log.info(req.dcaeReqId,"Cloudify is not healthy");
+                            res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                        }
+                    } else {
+                        log.info(req.dcaeReqId,"Cloudify is not healthy; responded with status " + cfyStatusResp.status);
+                        res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                    }
+                })
+        }
+        else {
+            res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+            log.info(req.dcaeReqId,"Inventory is not healthy");
+        }
+    })
+
+     /* All errors show up here */
+
+     .catch(function (error) {
+         /* If we haven't already sent a response, let the error handler send response and log the error */
+         if (!res.headersSent) {
+                next(error);
+         }
+         else {
+             /* Already sent the response, so just log error */
+             error.message = "Error checking service health :" + error.message
+                 + " " + (error.stack || "").replace(/\n/g, " ");
+             log.error(error, req);
+             log.audit(req, 500, error.message);
+         }
+     });
+});
+
+module.exports = app;
diff --git a/package.json b/package.json
index 9f24b9f..3bde401 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "onap-dcae-deployment-handler",
-  "version": "5.2.0",
+  "version": "6.0.0",
   "description": "ONAP DCAE Deployment Handler",
   "main": "deployment-handler.js",
   "dependencies": {
diff --git a/pom.xml b/pom.xml
index bf15408..05bda9e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
   <groupId>org.onap.dcaegen2.platform</groupId>
   <artifactId>deployment-handler</artifactId>
   <name>dcaegen2-platform-deployment-handler</name>
-  <version>3.2.0-SNAPSHOT</version>
+  <version>4.0.0-SNAPSHOT</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/tests/mock_deployment_handler.js b/tests/mock_deployment_handler.js
index 5ed1985..36f3ac4 100644
--- a/tests/mock_deployment_handler.js
+++ b/tests/mock_deployment_handler.js
@@ -29,7 +29,8 @@ const LOG_PATH = './log/';
 const CONSUL_URL = 'http://consul:8500';
 const MOCK_CLOUDIFY_MANAGER = "mock_cloudify_manager";
 const CLOUDIFY_URL = "http://" + MOCK_CLOUDIFY_MANAGER + ":80";
-const CLOUDIFY_API = "/api/v2.1";
+//const CLOUDIFY_API = "/api/v2.1";
+const CLOUDIFY_API = "/api/v3.1";
 
 const MOCK_INVENTORY = "mock_inventory";
 const INVENTORY_URL = "https://" + MOCK_INVENTORY + ":8080";
diff --git a/tests/test_service-health.js b/tests/test_service-health.js
new file mode 100644
index 0000000..a73cec1
--- /dev/null
+++ b/tests/test_service-health.js
@@ -0,0 +1,276 @@
+/*
+Copyright(c) 2019 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/**
+ * handling policy updates
+ */
+
+"use strict";
+
+const nock = require('nock')
+    , chai = require('chai')
+    , chaiHttp = require('chai-http')
+    , expect = chai.expect
+    , assert = chai.assert;
+
+chai.use(chaiHttp);
+
+const dh = require('./mock_deployment_handler');
+const utils = require('./mock_utils');
+
+const INV_PATH_DCAE_SERVICE_TYPES = "/dcae-service-types/";
+const INV_SERVICE_HEALTH = "/servicehealth";
+
+const Inventory = {
+    resp_empty: {"links":{"previousLink":null,"nextLink":null},"totalCount":0,"items":[]},
+    resp_service_types: function(service_type, totalCount) {
+        service_type = service_type || "f93264ee-348c-44f6-af3d-15b157bba735";
+        const res = {
+            "links": {
+                "previousLink": null,
+                "nextLink": {
+                    "rel": "next",
+                    "href": dh.INVENTORY_URL + INV_PATH_DCAE_SERVICE_TYPES + "?onlyLatest=true&onlyActive=true&offset=25"
+                }
+            },
+            "totalCount": totalCount || 1,
+            "items": []
+        };
+        Array.from(Array(totalCount || 1), (_, idx) => idx).forEach(index => {
+            //const dpl_id = deployment_id + ((index && "_" + index) || "");
+            res.items.push({
+                "owner": "dcaeorch",
+                "typeName": "svc-type-000",
+                "typeVersion": 1,
+                "blueprintTemplate": "tosca_definitions_version: cloudify_dsl_1_3\nimports:\n  - \"http://www.getcloudify.org/spec/cloudify/3.4/types.yaml\"\n  - https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R4/dockerplugin/3.2.0/dockerplugin_types.yaml\n  - https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R4/relationshipplugin/1.0.0/relationshipplugin_types.yaml\n  - https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R4/dcaepolicyplugin/2.3.0/dcaepolicyplugin_types.yaml\n\ninputs:\n  dh_override:\n    type: string\n    default: \"dockerhost\"\n  dh_location_id:\n    type: string\n    default: \"zone1\"\n  aaiEnrichmentHost:\n    type: string\n    default: \"none\"\n  aaiEnrichmentPort:\n    type: string    \n    default: 8443\n  enableAAIEnrichment:\n    type: string\n    default: false\n  dmaap_host:\n    type: string\n    default: dmaap.onap-message-router   \n  dmaap_port:\n    type: string\n    default: 3904    \n  enableRedisCaching:\n    type: string\n    default: false    \n  redisHosts:\n    type: string      \n  tag_version:\n    type: string\n    default: \"nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.tca-cdap-container:1.0.0\"\n  consul_host:\n    type: string\n    default: consul-server.onap-consul\n  consul_port:\n    type: string\n    default: \"8500\"\n  cbs_host:\n    type: string\n    default: \"config-binding-service.dcae\"\n  cbs_port:\n    type: string\n    default: \"10000\"\n  policy_id:\n    type: string\n    default: \"none\"\n  external_port:\n    type: string\n    description: \"Port for CDAPgui to be exposed\"\n    default: \"32010\"\n  scn_name: \n    default: dcaegen2-analytics_tca_clampinstance_1\n    type: string\nnode_templates:\n  docker_service_host:\n    properties:\n      docker_host_override:\n        get_input: dh_override\n      location_id:\n        get_input: dh_location_id\n    type: dcae.nodes.SelectedDockerHost\n  tca_docker:\n    relationships:\n       - type: dcae.relationships.component_contained_in\n         target: docker_service_host\n       - target: tca_policy\n         type: cloudify.relationships.depends_on        \n    type: dcae.nodes.DockerContainerForComponentsUsingDmaap\n    properties:\n        application_config:\n            app_config:\n                appDescription: DCAE Analytics Threshold Crossing Alert Application\n                appName: dcae-tca\n                tcaAlertsAbatementTableName: TCAAlertsAbatementTable\n                tcaAlertsAbatementTableTTLSeconds: '1728000'\n                tcaSubscriberOutputStreamName: TCASubscriberOutputStream\n                tcaVESAlertsTableName: TCAVESAlertsTable\n                tcaVESAlertsTableTTLSeconds: '1728000'\n                tcaVESMessageStatusTableName: TCAVESMessageStatusTable\n                tcaVESMessageStatusTableTTLSeconds: '86400'\n                thresholdCalculatorFlowletInstances: '2'\n            app_preferences:\n                aaiEnrichmentHost: \n                    get_input: aaiEnrichmentHost\n                aaiEnrichmentIgnoreSSLCertificateErrors: 'true'\n                aaiEnrichmentPortNumber: '8443'\n                aaiEnrichmentProtocol: https\n                aaiEnrichmentUserName: DCAE\n                aaiEnrichmentUserPassword: DCAE\n                aaiVMEnrichmentAPIPath: /aai/v11/search/nodes-query\n                aaiVNFEnrichmentAPIPath: /aai/v11/network/generic-vnfs/generic-vnf\n                enableAAIEnrichment: \n                    get_input: enableAAIEnrichment\n                enableRedisCaching: \n                    get_input: enableRedisCaching\n                redisHosts: \n                    get_input: redisHosts\n                enableAlertCEFFormat: 'false'\n                publisherContentType: application/json\n                publisherHostName: \n                    get_input: dmaap_host\n                publisherHostPort: \n                    get_input: dmaap_port                  \n                publisherMaxBatchSize: '1'\n                publisherMaxRecoveryQueueSize: '100000'\n                publisherPollingInterval: '20000'\n                publisherProtocol: http\n                publisherTopicName: unauthenticated.DCAE_CL_OUTPUT\n                subscriberConsumerGroup: OpenDCAE-c12\n                subscriberConsumerId: c12\n                subscriberContentType: application/json\n                subscriberHostName: \n                    get_input: dmaap_host\n                subscriberHostPort:\n                    get_input: dmaap_port                                  \n                subscriberMessageLimit: '-1'\n                subscriberPollingInterval: '30000'\n                subscriberProtocol: http\n                subscriberTimeoutMS: '-1'\n                subscriberTopicName: unauthenticated.SEC_MEASUREMENT_OUTPUT\n                tca_policy_default: '{\"domain\":\"measurementsForVfScaling\",\"metricsPerEventName\":[{\"eventName\":\"vFirewallBroadcastPackets\",\"controlLoopSchemaType\":\"VNF\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":300,\"direction\":\"LESS_OR_EQUAL\",\"severity\":\"MAJOR\",\"closedLoopEventStatus\":\"ONSET\"},{\"closedLoopControlName\":\"ControlLoop-vFirewall-d0a1dfc6-94f5-4fd4-a5b5-4630b438850a\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":700,\"direction\":\"GREATER_OR_EQUAL\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]},{\"eventName\":\"vLoadBalancer\",\"controlLoopSchemaType\":\"VM\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vDNS-6f37f56d-a87d-4b85-b6a9-cc953cf779b3\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.vNicUsageArray[*].receivedTotalPacketsDelta\",\"thresholdValue\":300,\"direction\":\"GREATER_OR_EQUAL\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]},{\"eventName\":\"Measurement_vGMUX\",\"controlLoopSchemaType\":\"VNF\",\"policyScope\":\"DCAE\",\"policyName\":\"DCAE.Config_tca-hi-lo\",\"policyVersion\":\"v0.0.1\",\"thresholds\":[{\"closedLoopControlName\":\"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value\",\"thresholdValue\":0,\"direction\":\"EQUAL\",\"severity\":\"MAJOR\",\"closedLoopEventStatus\":\"ABATED\"},{\"closedLoopControlName\":\"ControlLoop-vCPE-48f0c2c3-a172-4192-9ae3-052274181b6e\",\"version\":\"1.0.2\",\"fieldPath\":\"$.event.measurementsForVfScalingFields.additionalMeasurements[*].arrayOfFields[0].value\",\"thresholdValue\":0,\"direction\":\"GREATER\",\"severity\":\"CRITICAL\",\"closedLoopEventStatus\":\"ONSET\"}]}]}'\n        service_component_type: dcaegen2-analytics_tca    \n        docker_config:\n            healthcheck:\n               endpoint: /\n               interval: 15s\n               timeout: 1s\n               type: http\n        image:\n            get_input: tag_version   \n        service_component_name_override: \n            get_input: scn_name            \n    interfaces:\n      cloudify.interfaces.lifecycle:\n        start:\n          inputs:\n            envs:\n                DMAAPHOST: \n                    { get_input: dmaap_host }\n                DMAAPPORT:\n                    { get_input: dmaap_port }\n                DMAAPPUBTOPIC: \"unauthenticated.DCAE_CL_OUTPUT\"\n                DMAAPSUBTOPIC: \"unauthenticated.SEC_MEASUREMENT_OUTPUT\"\n                AAIHOST: \n                    { get_input: aaiEnrichmentHost }\n                AAIPORT: \n                    { get_input: aaiEnrichmentPort }\n                CONSUL_HOST: \n                    { get_input: consul_host }\n                CONSUL_PORT: \n                    { get_input: consul_port }\n                CBS_HOST: \n                    { get_input: cbs_host }\n                CBS_PORT: \n                    { get_input: cbs_port }\n                CONFIG_BINDING_SERVICE: \"config_binding_service\"                \n                SERVICE_11011_NAME: \n                    { get_input: scn_name }\n                SERVICE_11015_IGNORE: \"true\"                \n            ports:\n              - concat: [\"11011:\", { get_input: external_port }]        \n        stop:\n          inputs:\n            cleanup_image: true              \n  tca_policy:\n    type: dcae.nodes.policy\n    properties:\n      policy_id:\n           get_input: policy_id\n",
+                "serviceIds": null,
+                "vnfTypes": ["TESTVNF000"],
+                "serviceLocations": null,
+                "asdcServiceId": null,
+                "asdcResourceId": null,
+                "asdcServiceURL": null,
+                "typeId": service_type,
+                "selfLink": {
+                    "rel": "self",
+                    "href": dh.INVENTORY_URL + INV_PATH_DCAE_SERVICE_TYPES + service_type
+                },
+                "created": 1500910967567,
+                "deactivated": null
+            });
+        });
+        return res;
+    }
+
+};
+
+const Cloudify = {
+    resp_status: function() {
+        return {
+            "status": "running",
+            "services": [
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "Cloudify Composer Service",
+                            "state": "running",
+                            "MainPID": 25094,
+                            "Id": "cloudify-composer.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Cloudify Composer",
+                    "unit_id": "cloudify-composer.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "LSB: Starts Logstash as a daemon.",
+                            "state": "running",
+                            "MainPID": 0,
+                            "Id": "logstash.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Logstash",
+                    "unit_id": "logstash.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "RabbitMQ Service",
+                            "state": "running",
+                            "MainPID": 93479,
+                            "Id": "cloudify-rabbitmq.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "RabbitMQ",
+                    "unit_id": "cloudify-rabbitmq.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "PostgreSQL 9.5 database server",
+                            "state": "running",
+                            "MainPID": 70688,
+                            "Id": "cloudify-postgresql.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "PostgreSQL",
+                    "unit_id": "cloudify-postgresql.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "nginx - high performance web server",
+                            "state": "running",
+                            "MainPID": 114673,
+                            "Id": "nginx.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Webserver",
+                    "unit_id": "nginx.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "Cloudify Management Worker Service",
+                            "state": "running",
+                            "MainPID": 93818,
+                            "Id": "cloudify-mgmtworker.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Management Worker",
+                    "unit_id": "cloudify-mgmtworker.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "syncthing",
+                            "state": "running",
+                            "MainPID": 102764,
+                            "Id": "cloudify-syncthing.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Syncthing",
+                    "unit_id": "cloudify-syncthing.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "Cloudify Console Service",
+                            "state": "running",
+                            "MainPID": 25085,
+                            "Id": "cloudify-stage.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Cloudify Console",
+                    "unit_id": "cloudify-stage.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "Cloudify REST Service",
+                            "state": "running",
+                            "MainPID": 93233,
+                            "Id": "cloudify-restservice.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Manager Rest-Service",
+                    "unit_id": "cloudify-restservice.service"
+                },
+                {
+                    "instances": [
+                        {
+                            "LoadState": "loaded",
+                            "Description": "consul",
+                            "state": "running",
+                            "MainPID": 24394,
+                            "Id": "cloudify-consul.service",
+                            "ActiveState": "active",
+                            "SubState": "running"
+                        }
+                    ],
+                    "display_name": "Consul",
+                    "unit_id": "cloudify-consul.service"
+                }
+            ]
+        }
+    }
+};
+
+function test_service_health(dh_server) {
+    const req_path = "/servicehealth";
+    const test_txt = "GET " + req_path;
+    describe(test_txt, () => {
+        it('GET all the dcae-service-types from inventory', function() {
+            const action_timer = new utils.ActionTimer();
+            console.log(action_timer.step, test_txt);
+
+            //const inv_resp = Inventory.resp_service_types();
+            nock(dh.INVENTORY_URL).get(INV_SERVICE_HEALTH)
+                .reply(200, function(uri) {
+                    console.log(action_timer.step, "get", dh.INVENTORY_URL, uri);
+                    return JSON.stringify(Inventory.resp_service_types());
+                });
+
+            nock(dh.CLOUDIFY_URL).get(dh.CLOUDIFY_API + "/status")
+                .reply(200, function(uri) {
+                    console.log(action_timer.step, "get", dh.CLOUDIFY_URL, uri);
+                    return JSON.stringify(Cloudify.resp_status());
+                });
+
+            return chai.request(dh_server.app).get(req_path)
+                .then(function(res) {
+                    console.log(action_timer.step, "res for", test_txt, res.text);
+                    expect(res).to.have.status(200);
+                    expect(res).to.be.json;
+                })
+                .catch(function(err) {
+                    console.error(action_timer.step, "err for", test_txt, err);
+                    throw err;
+                });
+        });
+    });
+}
+
+
+
+dh.add_tests([test_service_health]);
diff --git a/version.js b/version.js
index e03ee1e..36f7332 100644
--- a/version.js
+++ b/version.js
@@ -1,21 +1,21 @@
-/*-
- * ============LICENSE_START=======================================================
- * PROJECT
- * ================================================================================
- * Copyright (C) 2018 AT&T Intellectual Property. All rights
- *                                              reserved.
- * ================================================================================
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * ============LICENSE_END=========================================================
- */
-exports.commit="unspecified";
+/*-

+ * ============LICENSE_START=======================================================

+ * PROJECT

+ * ================================================================================

+ * Copyright (C) 2018 AT&T Intellectual Property. All rights

+ *                                              reserved.

+ * ================================================================================

+ * Licensed under the Apache License, Version 2.0 (the "License");

+ * you may not use this file except in compliance with the License.

+ * You may obtain a copy of the License at

+ *

+ *      http://www.apache.org/licenses/LICENSE-2.0

+ *

+ * Unless required by applicable law or agreed to in writing, software

+ * distributed under the License is distributed on an "AS IS" BASIS,

+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+ * See the License for the specific language governing permissions and

+ * limitations under the License.

+ * ============LICENSE_END=========================================================

+ */

+exports.commit="unspecified";

diff --git a/version.properties b/version.properties
index f1f03dc..967829d 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
-major=3

-minor=2

-patch=0

-base_version=${major}.${minor}.${patch}

-release_version=${base_version}

-snapshot_version=${base_version}-SNAPSHOT

+major=4
+minor=0
+patch=0
+base_version=${major}.${minor}.${patch}
+release_version=${base_version}
+snapshot_version=${base_version}-SNAPSHOT