From 2832ba2fbd0738bc900af5599253e2f3bfe02a70 Mon Sep 17 00:00:00 2001 From: Jack Lucas Date: Fri, 20 Apr 2018 13:22:05 +0000 Subject: Add healthcheck container for OOM Change-Id: Ie4719a0e4705901fd9d0fa3504696fbefc6c704a Issue-ID: DCAEGEN2-461 Signed-off-by: Jack Lucas --- healthcheck-container/Dockerfile | 8 ++ healthcheck-container/get-status.js | 109 ++++++++++++++++++++++ healthcheck-container/healthcheck.js | 80 ++++++++++++++++ healthcheck-container/package.json | 11 +++ healthcheck-container/pom.xml | 172 +++++++++++++++++++++++++++++++++++ mvn-phase-script.sh | 2 +- pom.xml | 1 + 7 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 healthcheck-container/Dockerfile create mode 100644 healthcheck-container/get-status.js create mode 100644 healthcheck-container/healthcheck.js create mode 100644 healthcheck-container/package.json create mode 100644 healthcheck-container/pom.xml diff --git a/healthcheck-container/Dockerfile b/healthcheck-container/Dockerfile new file mode 100644 index 0000000..d1b4231 --- /dev/null +++ b/healthcheck-container/Dockerfile @@ -0,0 +1,8 @@ +FROM node:8.11.1 +RUN mkdir -p /opt/app +COPY *.js /opt/app/ +COPY package.json /opt/app/ +WORKDIR /opt/app +RUN npm install --only=production +EXPOSE 80 +ENTRYPOINT ["/usr/local/bin/node", "healthcheck.js"] diff --git a/healthcheck-container/get-status.js b/healthcheck-container/get-status.js new file mode 100644 index 0000000..2ed1d3d --- /dev/null +++ b/healthcheck-container/get-status.js @@ -0,0 +1,109 @@ +/* +Copyright(c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. +*/ + +/* + * Query Kubernetes for status of deployments and extract readiness information + */ + +const fs = require('fs'); +const request = require('request'); + +const K8S_CREDS = '/var/run/secrets/kubernetes.io/serviceaccount'; +const K8S_API = 'https://kubernetes.default.svc.cluster.local/'; // Full name to match cert for TLS +const K8S_PATH = 'apis/apps/v1beta2/namespaces/'; + +//Get token and CA cert +const ca = fs.readFileSync(K8S_CREDS + '/ca.crt'); +const token = fs.readFileSync(K8S_CREDS + '/token'); + +const summarizeDeploymentList = function(list) { + // list is a DeploymentList object returned by k8s + // Individual deployments are in the array 'items' + + let ret = + { + type: "summary", + count: 0, + ready: 0, + items: [] + }; + + // Extract readiness information + for (let deployment of list.items) { + ret.items.push( + { + name: deployment.metadata.name, + ready: deployment.status.readyReplicas || 0, + unavailable: deployment.status.unavailableReplicas || 0 + } + ); + ret.count ++; + ret.ready = ret.ready + (deployment.status.readyReplicas || 0); + } + + return ret; +}; + +const summarizeDeployment = function(deployment) { + // deployment is a Deployment object returned by k8s + // we make it look enough like a DeploymentList object to + // satisfy summarizeDeploymentList + return summarizeDeploymentList({items: [deployment]}); +}; + +const queryKubernetes = function(path, callback) { + // Make request to Kubernetes + + const options = { + url: K8S_API + path, + ca : ca, + headers: { + Authorization: 'bearer ' + token + }, + json: true + }; + console.log ("request url: " + options.url); + request(options, function(error, res, body) { + console.log ("status: " + (res && res.statusCode) ? res.statusCode : "NONE"); + if (error) { + console.log("error: " + error); + } + callback(error, res, body); + }); +}; + +const getStatus = function(path, extract, callback) { + // Get info from k8s and extract readiness info + queryKubernetes(path, function(error, res, body) { + let ret = body; + if (!error && res && res.statusCode === 200) { + ret = extract(body); + } + callback (error, res, ret); + }); +}; + +exports.getStatusNamespace = function (namespace, callback) { + // Get readiness information for all deployments in namespace + const path = K8S_PATH + namespace + '/deployments'; + getStatus(path, summarizeDeploymentList, callback); +}; + +exports.getStatusSingle = function (namespace, deployment, callback) { + // Get readiness information for a single deployment + const path = K8S_PATH + namespace + '/deployments/' + deployment; + getStatus(path, summarizeDeployment, callback); +}; \ No newline at end of file diff --git a/healthcheck-container/healthcheck.js b/healthcheck-container/healthcheck.js new file mode 100644 index 0000000..7555032 --- /dev/null +++ b/healthcheck-container/healthcheck.js @@ -0,0 +1,80 @@ +/* +Copyright(c) 2018 AT&T Intellectual Property. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. +*/ + +//Expect ONAP and DCAE namespaces and Helm "release" name to be passed via environment variables +// +const ONAP_NS = process.env.ONAP_NAMESPACE || 'default'; +const DCAE_NS = process.env.DCAE_NAMESPACE || 'default'; +const HELM_REL = process.env.HELM_RELEASE || ''; + +const HEALTHY = 200; +const UNHEALTHY = 500; +const UNKNOWN = 503; + +const status = require('./get-status'); +const http = require('http'); + +const isHealthy = function(summary) { + // Current healthiness criterion is simple--all deployments are ready + return summary.count && summary.ready && summary.count === summary.ready; +}; + +const checkHealth = function (callback) { + // Makes queries to Kubernetes and checks results + // If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500) + // If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503) + // If we get responses from k8s and all deployments are ready, health status is HEALTHY (200) + // This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking + status.getStatusNamespace(DCAE_NS, function(err, res, body) { + let ret = {status : UNKNOWN, body: [body]}; + if (err) { + callback(ret); + } + else if (body.type && body.type === 'summary') { + if (isHealthy(body)) { + // All the DCAE components report healthy -- check Cloudify Manager + let cmDeployment = 'dcae-cloudify-manager'; + if (HELM_REL.length > 0) { + cmDeployment = HELM_REL + '-' + cmDeployment; + } + status.getStatusSingle(ONAP_NS, cmDeployment, function (err, res, body){ + ret.body.push(body); + if (err) { + callback(ret); + } + if (body.type && body.type === 'summary') { + ret.status = isHealthy(body) ? HEALTHY : UNHEALTHY; + } + callback(ret); + }); + } + else { + callback(ret); + } + } + }); +}; + +// Simple HTTP server--any incoming request triggers a health check +const server = http.createServer(function(req, res) { + checkHealth(function(ret) { + console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret)); + res.statusCode = ret.status; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify(ret.body || {}), 'utf8'); + }); +}); +server.listen(80); diff --git a/healthcheck-container/package.json b/healthcheck-container/package.json new file mode 100644 index 0000000..2a08bdd --- /dev/null +++ b/healthcheck-container/package.json @@ -0,0 +1,11 @@ +{ + "name": "k8s-healthcheck", + "description": "DCAE healthcheck server", + "version": "1.0.0", + "main": "healthcheck.js", + "dependencies": { + "request": "2.85.0" + }, + "author": "author", + "license": "(Apache-2.0)" +} diff --git a/healthcheck-container/pom.xml b/healthcheck-container/pom.xml new file mode 100644 index 0000000..dea3c48 --- /dev/null +++ b/healthcheck-container/pom.xml @@ -0,0 +1,172 @@ + + + + 4.0.0 + + org.onap.dcaegen2.deployments + deployments + 1.2.0-SNAPSHOT + + org.onap.dcaegen2.deployments + healthcheck-container + dcaegen2-deployments-healthcheck-container + 1.0.0 + http://maven.apache.org + + UTF-8 + true + . + + + + + py + Python + **/*.py + + + + + ${project.artifactId}-${project.version} + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + clean phase script + clean + + exec + + + + ${project.artifactId} + clean + + + + + generate-sources script + generate-sources + + exec + + + + ${project.artifactId} + generate-sources + + + + + compile script + compile + + exec + + + + ${project.artifactId} + compile + + + + + package script + package + + exec + + + + ${project.artifactId} + package + + + + + test script + test + + exec + + + + ${project.artifactId} + test + + + + + install script + install + + exec + + + + ${project.artifactId} + install + + + + + deploy script + deploy + + exec + + + + ${project.artifactId} + deploy + + + + + + + + diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh index 9897d5e..acada60 100755 --- a/mvn-phase-script.sh +++ b/mvn-phase-script.sh @@ -83,7 +83,7 @@ deploy) upload_files_of_extension sh build_and_push_docker ;; - k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container) + k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container|healthcheck-container) build_and_push_docker ;; scripts|cloud_init|heat) diff --git a/pom.xml b/pom.xml index ce585ba..ae1f1b9 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,7 @@ limitations under the License. cm-container k8s-bootstrap-container tca-cdap-container + healthcheck-container -- cgit 1.2.3-korg