summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--healthcheck-container/Dockerfile1
-rw-r--r--healthcheck-container/README.md103
-rw-r--r--healthcheck-container/get-status.js228
-rw-r--r--healthcheck-container/healthcheck.js79
-rw-r--r--healthcheck-container/package.json2
-rw-r--r--healthcheck-container/pom.xml2
6 files changed, 268 insertions, 147 deletions
diff --git a/healthcheck-container/Dockerfile b/healthcheck-container/Dockerfile
index 10a6488..ac61e1b 100644
--- a/healthcheck-container/Dockerfile
+++ b/healthcheck-container/Dockerfile
@@ -21,6 +21,5 @@ RUN mkdir -p /opt/app
COPY *.js /opt/app/
COPY package.json /opt/app/
WORKDIR /opt/app
-RUN npm install --only=production
EXPOSE 80
ENTRYPOINT ["/usr/local/bin/node", "healthcheck.js"]
diff --git a/healthcheck-container/README.md b/healthcheck-container/README.md
new file mode 100644
index 0000000..fb8d87c
--- /dev/null
+++ b/healthcheck-container/README.md
@@ -0,0 +1,103 @@
+# DCAE Healthcheck Service
+
+The DCAE Healthcheck service provides a simple HTTP API to check the status of DCAE components running in the Kubernetes environment. When it receives any incoming HTTP request, the service makes queries to the Kubernetes API to determine the current status of the DCAE components, as seen by Kubernetes. Most components have defined a "readiness probe" (an HTTP healthcheck endpoint or a healthcheck script) that Kubernetes uses to
+determine readiness.
+
+The Healthcheck service has three sources for identifying components that should be running:
+1. A hardcoded list of components that are expected to be deployed by Helm as part of the ONAP installation.
+2. A hardcoded list of components thar are expected to be deployed with blueprints using Cloudify Manager during DCAE bootstrapping, which is part of ONAP installation.
+3. Components labeled in Kubernetes as having been deployed by Cloudify Manager. These are identified by a query to the Kubernetes API. The query is made each time an incoming HTTP request is made.
+
+Note that by "component", we mean a Kubernetes Deployment object associated with the component.
+
+Sources 2 and 3 are likely to overlap (the components in source 2 are labeled in Kubernetes as having been deployed by Cloudify Manager, so they will show up as part of source 3 if the bootstrap process progressed to the point of attempting deployments). The code de-duplicates these sources.
+
+The Healthcheck service returns an HTTP status code of 200 if Kubernetes reports that all of the components that should be running are in a ready state. It returns a status code of 500 if some of the components are not ready. It returns a status code of 503 if some kind of error prevented it from completing a query.
+
+For the 200 and 500 status codes, the Healthcheck service returns a body consisting of a JSON object. The object has the following fields:
+- `type` : the type of response, currently always set to `summary`.
+- `count`: the total number of deployments that have been checked.
+- `ready`: the number of deployments in the ready state
+- `items`: a JSON list(array) of objects, one for each deployment. Each object has the form:
+`{"name": "k8s_deployment_name", "ready": number_of_ready_instances, "unavailable": number_number_of_unavailable_instances}`
+
+Here's an example of the body, with one component in an unavailable state:
+```
+{
+ "type": "summary",
+ "count": 14,
+ "ready": 13,
+ "items": [
+ {
+ "name": "dev-dcaegen2-dcae-cloudify-manager",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-config-binding-service",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-deployment-handler",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-inventory",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-service-change-handler",
+ "ready": 0,
+ "unavailable": 1
+ },
+ {
+ "name": "dep-policy-handler",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-ves-collector",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-tca-analytics",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-prh",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-hv-ves-collector",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-datafile-collector",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-dcae-snmptrap-collector",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-holmes-engine-mgmt",
+ "ready": 1,
+ "unavailable": 0
+ },
+ {
+ "name": "dep-holmes-rule-mgmt",
+ "ready": 1,
+ "unavailable": 0
+ }
+ ]
+}
+``` \ No newline at end of file
diff --git a/healthcheck-container/get-status.js b/healthcheck-container/get-status.js
index 9c4a723..565c1e0 100644
--- a/healthcheck-container/get-status.js
+++ b/healthcheck-container/get-status.js
@@ -33,141 +33,141 @@ const ca = fs.readFileSync(K8S_CREDS + '/ca.crt');
const token = fs.readFileSync(K8S_CREDS + '/token');
const summarizeDeploymentList = function(list) {
- // list is a DeploymentList object returned by k8s
- // Individual deployments are in the array 'items'
-
- let ret =
- {
- type: "summary",
- count: 0,
- ready: 0,
- items: []
- };
-
- // Extract readiness information
- for (let deployment of list.items) {
- ret.items.push(
- {
- name: deployment.metadata.name,
- ready: deployment.status.readyReplicas || 0,
- unavailable: deployment.status.unavailableReplicas || 0
- }
- );
- ret.count ++;
- ret.ready = ret.ready + (deployment.status.readyReplicas || 0);
- }
-
- return ret;
+ // list is a DeploymentList object returned by k8s
+ // Individual deployments are in the array 'items'
+
+ let ret =
+ {
+ type: "summary",
+ count: 0,
+ ready: 0,
+ items: []
+ };
+
+ // Extract readiness information
+ for (let deployment of list.items) {
+ ret.items.push(
+ {
+ name: deployment.metadata.name,
+ ready: deployment.status.readyReplicas || 0,
+ unavailable: deployment.status.unavailableReplicas || 0
+ }
+ );
+ ret.count ++;
+ ret.ready = ret.ready + (deployment.status.readyReplicas || 0);
+ }
+
+ return ret;
};
const summarizeDeployment = function(deployment) {
- // deployment is a Deployment object returned by k8s
- // we make it look enough like a DeploymentList object to
- // satisfy summarizeDeploymentList
- return summarizeDeploymentList({items: [deployment]});
+ // deployment is a Deployment object returned by k8s
+ // we make it look enough like a DeploymentList object to
+ // satisfy summarizeDeploymentList
+ return summarizeDeploymentList({items: [deployment]});
};
const queryKubernetes = function(path, callback) {
- // Make GET request to Kubernetes API
- const options = {
- host: K8S_HOST,
- path: "/" + path,
- ca : ca,
- headers: {
- Authorization: 'bearer ' + token
- }
- };
- console.log ("request url: " + options.host + options.path);
- const req = https.get(options, function(resp) {
- let rawBody = "";
- resp.on("data", function(data) {
- rawBody += data;
- });
- resp.on("error", function (error) {
- console.error("error: " + error);
- callback(error, null, null)
- });
- resp.on("end", function() {
- console.log ("status: " + resp.statusCode ? resp.statusCode: "NONE")
- callback(null, resp, JSON.parse(rawBody));
- });
- });
- req.end();
+ // Make GET request to Kubernetes API
+ const options = {
+ host: K8S_HOST,
+ path: "/" + path,
+ ca : ca,
+ headers: {
+ Authorization: 'bearer ' + token
+ }
+ };
+ console.log ("request url: " + options.host + options.path);
+ const req = https.get(options, function(resp) {
+ let rawBody = "";
+ resp.on("data", function(data) {
+ rawBody += data;
+ });
+ resp.on("error", function (error) {
+ console.error("error: " + error);
+ callback(error, null, null)
+ });
+ resp.on("end", function() {
+ console.log ("status: " + resp.statusCode ? resp.statusCode: "NONE")
+ callback(null, resp, JSON.parse(rawBody));
+ });
+ });
+ req.end();
};
const getStatus = function(path, extract, callback) {
- // Get info from k8s and extract readiness info
- queryKubernetes(path, function(error, res, body) {
- let ret = body;
- if (!error && res && res.statusCode === 200) {
- ret = extract(body);
- }
- callback (error, res, ret);
- });
+ // Get info from k8s and extract readiness info
+ queryKubernetes(path, function(error, res, body) {
+ let ret = body;
+ if (!error && res && res.statusCode === 200) {
+ ret = extract(body);
+ }
+ callback (error, res, ret);
+ });
};
const getStatusSinglePromise = function (item) {
- // Expect item to be of the form {namespace: "namespace", deployment: "deployment_name"}
- return new Promise(function(resolve, reject){
- const path = K8S_PATH + item.namespace + '/deployments/' + item.deployment;
- queryKubernetes(path, function(error, res, body){
- if (error) {
- reject(error);
- }
- else if (res.statusCode === 404) {
- // Treat absent deployment as if it's an unhealthy deployment
- resolve ({
- metadata: {name: item.deployment},
- status: {unavailableReplicas: 1}
- });
- }
- else if (res.statusCode != 200) {
- reject(body);
- }
- else {
- resolve(body);
- }
- });
- });
+ // Expect item to be of the form {namespace: "namespace", deployment: "deployment_name"}
+ return new Promise(function(resolve, reject){
+ const path = K8S_PATH + item.namespace + '/deployments/' + item.deployment;
+ queryKubernetes(path, function(error, res, body){
+ if (error) {
+ reject(error);
+ }
+ else if (res.statusCode === 404) {
+ // Treat absent deployment as if it's an unhealthy deployment
+ resolve ({
+ metadata: {name: item.deployment},
+ status: {unavailableReplicas: 1}
+ });
+ }
+ else if (res.statusCode != 200) {
+ reject(body);
+ }
+ else {
+ resolve(body);
+ }
+ });
+ });
}
exports.getStatusNamespace = function (namespace, callback) {
- // Get readiness information for all deployments in namespace
- const path = K8S_PATH + namespace + '/deployments';
- getStatus(path, summarizeDeploymentList, callback);
+ // Get readiness information for all deployments in namespace
+ const path = K8S_PATH + namespace + '/deployments';
+ getStatus(path, summarizeDeploymentList, callback);
};
exports.getStatusSingle = function (namespace, deployment, callback) {
- // Get readiness information for a single deployment
- const path = K8S_PATH + namespace + '/deployments/' + deployment;
- getStatus(path, summarizeDeployment, callback);
+ // Get readiness information for a single deployment
+ const path = K8S_PATH + namespace + '/deployments/' + deployment;
+ getStatus(path, summarizeDeployment, callback);
};
exports.getStatusListPromise = function (list) {
- // List is of the form [{namespace: "namespace", deployment: "deployment_name"}, ... ]
- const p = Promise.all(list.map(getStatusSinglePromise))
- return p.then(function(results) {
- return summarizeDeploymentList({items: results});
- });
+ // List is of the form [{namespace: "namespace", deployment: "deployment_name"}, ... ]
+ const p = Promise.all(list.map(getStatusSinglePromise))
+ return p.then(function(results) {
+ return summarizeDeploymentList({items: results});
+ });
}
exports.getDCAEDeploymentsPromise = function (namespace) {
- // Return list of the form [{namespace: "namespace"}, deployment: "deployment_name"].
- // List contains all k8s deployments in the specified namespace that were deployed
- // by Cloudify, based on Cloudify's use of a "marker" label on each k8s deployment that
- // the k8s plugin created.
-
- return new Promise(function(resolve, reject) {
- const path = K8S_PATH + namespace + '/deployments?labelSelector=' + CFY_LABEL + '&limit=' + MAX_DEPS
- queryKubernetes(path, function(error, res, body){
- if (error) {
- reject(error);
- }
- else if (res.statusCode !== 200) {
- reject(body);
- }
- else {
- resolve(body.items.map(function(i) {return {namespace : namespace, deployment: i.metadata.name};}));
- }
- });
- });
+ // Return list of the form [{namespace: "namespace"}, deployment: "deployment_name"].
+ // List contains all k8s deployments in the specified namespace that were deployed
+ // by Cloudify, based on Cloudify's use of a "marker" label on each k8s deployment that
+ // the k8s plugin created.
+
+ return new Promise(function(resolve, reject) {
+ const path = K8S_PATH + namespace + '/deployments?labelSelector=' + CFY_LABEL + '&limit=' + MAX_DEPS
+ queryKubernetes(path, function(error, res, body){
+ if (error) {
+ reject(error);
+ }
+ else if (res.statusCode !== 200) {
+ reject(body);
+ }
+ else {
+ resolve(body.items.map(function(i) {return {namespace : namespace, deployment: i.metadata.name};}));
+ }
+ });
+ });
};
diff --git a/healthcheck-container/healthcheck.js b/healthcheck-container/healthcheck.js
index a1c45e9..7a03f1e 100644
--- a/healthcheck-container/healthcheck.js
+++ b/healthcheck-container/healthcheck.js
@@ -24,52 +24,71 @@ const UNHEALTHY = 500;
const UNKNOWN = 503;
// List of deployments expected to be created via Helm
-const helmDeps =
- [
- 'dcae-cloudify-manager'
- ];
+const helmDeps =
+ [
+ 'dcae-cloudify-manager'
+ ];
+
+// List of deployments expected to be created by CM at boot time
+const bootDeps =
+ [
+ 'dep-config-binding-service',
+ 'dep-deployment-handler',
+ 'dep-inventory',
+ 'dep-service-change-handler',
+ 'dep-policy-handler',
+ 'dep-dcae-ves-collector',
+ 'dep-dcae-tca-analytics',
+ 'dep-dcae-prh',
+ 'dep-dcae-hv-ves-collector',
+ 'dep-dcae-datafile-collector'
+ ];
const status = require('./get-status');
const http = require('http');
// Helm deployments are always in the ONAP namespace and prefixed by Helm release name
const helmList = helmDeps.map(function(name) {
- return {namespace: ONAP_NS, deployment: HELM_REL.length > 0 ? HELM_REL + '-' + name : name};
+ return {namespace: ONAP_NS, deployment: HELM_REL.length > 0 ? HELM_REL + '-' + name : name};
});
const isHealthy = function(summary) {
- // Current healthiness criterion is simple--all deployments are ready
- return summary.count && summary.ready && summary.count === summary.ready;
+ // Current healthiness criterion is simple--all deployments are ready
+ return summary.count && summary.ready && summary.count === summary.ready;
};
const checkHealth = function (callback) {
- // Makes queries to Kubernetes and checks results
- // If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500)
- // If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503)
- // If we get responses from k8s and all deployments are ready, health status is HEALTHY (200)
- // This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking
+ // Makes queries to Kubernetes and checks results
+ // If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500)
+ // If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503)
+ // If we get responses from k8s and all deployments are ready, health status is HEALTHY (200)
+ // This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking
- // Query k8s to find all the deployments launched by CM (they all have a 'cfydeployment' label)
- status.getDCAEDeploymentsPromise(DCAE_NS)
- .then(function(dcaeList) {
- // Now get status for Helm deployments and CM deployments
- return status.getStatusListPromise(helmList.concat(dcaeList));
- })
- .then(function(body) {
- callback({status: isHealthy(body) ? HEALTHY : UNHEALTHY, body: body});
- })
- .catch(function(error){
- callback({status: UNKNOWN, body: [error]})
- });
+ // Query k8s to find all the deployments launched by CM (they all have a 'cfydeployment' label)
+ status.getDCAEDeploymentsPromise(DCAE_NS)
+ .then(function(fullDCAEList) {
+ // Remove any expected boot-time CM deployments from the list to avoid duplicates
+ dynamicDCAEDeps = fullDCAEList.filter(function(i) {return !(bootDeps.includes(i.deployment));})
+ // Create full list of CM deployments to check: boot deployments and anything else created by CM
+ dcaeList = (bootDeps.map(function(name){return {namespace: DCAE_NS, deployment: name}})).concat(dynamicDCAEDeps);
+ // Now get status for Helm deployments and CM deployments
+ return status.getStatusListPromise(helmList.concat(dcaeList));
+ })
+ .then(function(body) {
+ callback({status: isHealthy(body) ? HEALTHY : UNHEALTHY, body: body});
+ })
+ .catch(function(error){
+ callback({status: UNKNOWN, body: [error]})
+ });
};
// Simple HTTP server--any incoming request triggers a health check
const server = http.createServer(function(req, res) {
- checkHealth(function(ret) {
- console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret));
- res.statusCode = ret.status;
- res.setHeader('Content-Type', 'application/json');
- res.end(JSON.stringify(ret.body || {}), 'utf8');
- });
+ checkHealth(function(ret) {
+ console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret));
+ res.statusCode = ret.status;
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify(ret.body || {}), 'utf8');
+ });
});
server.listen(80);
diff --git a/healthcheck-container/package.json b/healthcheck-container/package.json
index 125bc24..674d382 100644
--- a/healthcheck-container/package.json
+++ b/healthcheck-container/package.json
@@ -1,7 +1,7 @@
{
"name": "k8s-healthcheck",
"description": "DCAE healthcheck server",
- "version": "1.2.0",
+ "version": "1.2.1",
"main": "healthcheck.js",
"author": "author",
"license": "(Apache-2.0)"
diff --git a/healthcheck-container/pom.xml b/healthcheck-container/pom.xml
index a0141a0..1f024e0 100644
--- a/healthcheck-container/pom.xml
+++ b/healthcheck-container/pom.xml
@@ -27,7 +27,7 @@ limitations under the License.
<groupId>org.onap.dcaegen2.deployments</groupId>
<artifactId>healthcheck-container</artifactId>
<name>dcaegen2-deployments-healthcheck-container</name>
- <version>1.2.0</version>
+ <version>1.2.1</version>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>