Add health and service endpoints

4.0.0 deployment-handler
Upgraded cloudify api from 2.1 to 3.1 due to new cloudify /status api
Requires upgrading Cloudify to 4.x
Added 2 APIs to check health of DH itself and to Inventory and Cloudify
Coverage summary
Statements   : 80.75% ( 994/1231 )
Branches     : 57.74% ( 302/523 )
Functions    : 81.13% ( 172/212 )
Lines        : 81.26% ( 984/1211 )

Issue-ID: DCAEGEN2-1127

Change-Id: Ic851a576eed668016f6cfe956f0f1880a65e1065
Signed-off-by: Shadi Haidar <sh1986@att.com>

5 files changed, 157 insertions, 9 deletions

diff --git a/lib/cloudify.js b/lib/cloudify.js
index 1cd5489..49c26dd 100644
--- a/lib/cloudify.js
+++ b/lib/cloudify.js
@@ -318,6 +318,16 @@ exports.getOutputDescriptions = function(req, dpid) {
     return doRequest(req, reqOptions, null, CLOUDIFY);
 };
 
+exports.getCfyStatus = function(req) {
+    const reqOptions = {
+        method : "GET",
+        uri : cfyAPI + "/status"
+    };
+    addAuthToOptions(reqOptions, req);
+
+    return doRequest(req, reqOptions, null, CLOUDIFY);
+};
+
 // Deletes a deployment
 exports.deleteDeployment = function(req, dpid) {
     const reqOptions = {
diff --git a/lib/config.js b/lib/config.js
index d4fd3e3..6430e36 100644
--- a/lib/config.js
+++ b/lib/config.js
@@ -1,5 +1,5 @@
 /*
-Copyright(c) 2017-2018 AT&T Intellectual Property. All rights reserved.
+Copyright(c) 2017-2019 AT&T Intellectual Property. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -79,7 +79,7 @@ const CONFIG_KEY = "deployment_handler";	/* Configuration is stored under the na
 const CM_NAME = "cloudify_manager";
 const INV_NAME = "inventory";
 
-const CM_API_PATH = "/api/v2.1";
+const CM_API_PATH = "/api/v3.1";
 const INV_API_PATH = "";
 
 const DEFAULT_CLOUDIFY_PROTOCOL = "https";
@@ -227,7 +227,6 @@ exports.configure = function() {
 		const missing = findMissingConfig(config);
 		if (missing.length > 0) {
 			throw new Error ("Required configuration elements missing: " + missing.join(','));
-			config = null;
 		}
 		console.log((new Date()) + ": config -> " + JSON.stringify(config, utils.hideSecrets, 2));
 		return config;
diff --git a/lib/deploy.js b/lib/deploy.js
index 4829040..cb78f6f 100644
--- a/lib/deploy.js
+++ b/lib/deploy.js
@@ -31,12 +31,7 @@ const createError = require('./dispatcher-error').createDispatcherError;
 var logger = require("./logging").getLogger();
 
 /* Set up the Cloudify low-level interface library */
-var cfy = require("./cloudify.js");
-/* Set config for interface library */
-cfy.setAPIAddress(config.cloudify.url);
-cfy.setCredentials(config.cloudify.user, config.cloudify.password);
-cfy.setLogger(logger);
-
+const cfy = process.mainModule.exports.cfy;
 
 
 
diff --git a/lib/inventory.js b/lib/inventory.js
index 5935067..0d06856 100644
--- a/lib/inventory.js
+++ b/lib/inventory.js
@@ -24,6 +24,7 @@ const createError = require('./dispatcher-error').createDispatcherError;
 
 const INV_SERV_TYPES = '/dcae-service-types';
 const INV_SERVICES = '/dcae-services';
+const INV_SERVICE_HEALTH = '/servicehealth';
 
 /*
  * Common error handling for inventory API calls
@@ -162,6 +163,29 @@ module.exports = function(options) {
                     return invError(err);
                 }
             });
+        },
+
+        /*
+        * Check if inventory service is healthy using inventory's service check api
+        */
+
+        isServiceHealthy: function(req) {
+            return doRequest(req, {
+                method: "GET",
+                uri: url + INV_SERVICE_HEALTH
+            }, null, INVENTORY)
+
+            .then(function(res) {
+                if ( res.status == 200 ) {
+                    return true;
+                }
+                else {
+                   return false;
+                }
+            },
+            function (err) {
+                return invError(err);
+            });
         }
     };
 };
diff --git a/lib/service-health.js b/lib/service-health.js
new file mode 100644
index 0000000..0b36b88
--- /dev/null
+++ b/lib/service-health.js
@@ -0,0 +1,120 @@
+/*
+Copyright(c) 2019 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/* Handle the /servicehealth API which checks if inventory and cloudify are healthy*/
+
+"use strict";
+
+const app = require('express')();
+app.set('x-powered-by', false);
+app.set('etag', false);
+
+const bodyParser = require('body-parser');
+const inv = require('./inventory');
+const log = require('./logging').getLogger();
+
+/* Pick up config and cloudify exported by main */
+const config = process.mainModule.exports.config;
+const cfy = process.mainModule.exports.cfy;
+const inventory = inv({url: config.inventory.url});
+
+app.use(bodyParser.json({strict: true}));				// Parse body as JSON
+app.use(function (req, res, next) {
+    log.info(req.dcaeReqId,
+        "new req: " + req.method + " " + req.originalUrl +
+        " from: " + req.ip + " body: " + JSON.stringify(req.body)
+    );
+    next();
+});
+
+
+/* Accept an incoming service health check request */
+app.get('/', function (req, res, next) {
+
+    /* Verify inventory service health*/
+    inventory.isServiceHealthy(req)
+
+    .then(function (isInvHealthy) {
+        log.info(req.dcaeReqId,"Checking isInventoryHealthy: " + isInvHealthy);
+        if ( isInvHealthy === true) {
+            log.info(req.dcaeReqId,"Inventory is healthy: ");
+            return cfy.getCfyStatus(req)
+
+                .then(function (cfyStatusResp) {
+                    log.info(req.dcaeReqId,"getCfyStatus Response -> " + JSON.stringify(cfyStatusResp, 2));
+                    if (cfyStatusResp.status === 200) {
+                        var isCfyHealthy = true;
+                        if ( cfyStatusResp.json && cfyStatusResp.json.status === "running" ) {
+                            log.info(req.dcaeReqId,"getCfyStatus Response status is running: ");
+                            const services = Object.keys(cfyStatusResp.json.services);
+                            log.info(req.dcaeReqId,"getCfyStatus services.length -> " + services.length);
+                            // run through all the cfy services to see which ones are running
+                            // don't stop looping when a cfy service is down, it's better to see a list of all serivces' status
+                            // so that we an idea about overall health of cfy
+                            services.forEach( service => {
+                                const service_display_name = cfyStatusResp.json.services[service].display_name;
+                                const instances = Object.keys(cfyStatusResp.json.services[service].instances);
+                                instances.forEach( instance => {
+                                    var description = cfyStatusResp.json.services[service].instances[instance].Description;
+                                    var subState = cfyStatusResp.json.services[service].instances[instance].SubState;
+                                    log.info(req.dcaeReqId,"cfy status for service display_name: " + service_display_name
+                                        + " and Description: " + description + " has a SubState of: " + subState);
+                                    if ( subState !== "running" ) {
+                                        log.info(req.dcaeReqId,"getCfyStatus Description-> " + description + " NOT running ok");
+                                        isCfyHealthy = false;
+                                        res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                                    }
+                                });
+                            });
+                        }
+                        if ( isCfyHealthy === true ) {
+                            log.info(req.dcaeReqId,"Cloudify is healthy");
+                            res.status(200).json({requestId: req.dcaeReqId, status: 'OK'});
+                        }
+                        else {
+                            log.info(req.dcaeReqId,"Cloudify is not healthy");
+                            res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                        }
+                    } else {
+                        log.info(req.dcaeReqId,"Cloudify is not healthy; responded with status " + cfyStatusResp.status);
+                        res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+                    }
+                })
+        }
+        else {
+            res.status(503).json({requestId: req.dcaeReqId, status: 'NOT OK'});
+            log.info(req.dcaeReqId,"Inventory is not healthy");
+        }
+    })
+
+     /* All errors show up here */
+
+     .catch(function (error) {
+         /* If we haven't already sent a response, let the error handler send response and log the error */
+         if (!res.headersSent) {
+                next(error);
+         }
+         else {
+             /* Already sent the response, so just log error */
+             error.message = "Error checking service health :" + error.message
+                 + " " + (error.stack || "").replace(/\n/g, " ");
+             log.error(error, req);
+             log.audit(req, 500, error.message);
+         }
+     });
+});
+
+module.exports = app;