summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJack Lucas <jflucas@research.att.com>2018-04-20 13:22:05 +0000
committerJack Lucas <jflucas@research.att.com>2018-04-20 13:22:58 +0000
commit2832ba2fbd0738bc900af5599253e2f3bfe02a70 (patch)
tree186a0d10a7fa0af1ca106c47fdcb58965750621b
parent2add4828edf3ceb7ce60e511093dd4fc25e0bb45 (diff)
Add healthcheck container for OOM
Change-Id: Ie4719a0e4705901fd9d0fa3504696fbefc6c704a Issue-ID: DCAEGEN2-461 Signed-off-by: Jack Lucas <jflucas@research.att.com>
-rw-r--r--healthcheck-container/Dockerfile8
-rw-r--r--healthcheck-container/get-status.js109
-rw-r--r--healthcheck-container/healthcheck.js80
-rw-r--r--healthcheck-container/package.json11
-rw-r--r--healthcheck-container/pom.xml172
-rwxr-xr-xmvn-phase-script.sh2
-rw-r--r--pom.xml1
7 files changed, 382 insertions, 1 deletions
diff --git a/healthcheck-container/Dockerfile b/healthcheck-container/Dockerfile
new file mode 100644
index 0000000..d1b4231
--- /dev/null
+++ b/healthcheck-container/Dockerfile
@@ -0,0 +1,8 @@
+FROM node:8.11.1
+RUN mkdir -p /opt/app
+COPY *.js /opt/app/
+COPY package.json /opt/app/
+WORKDIR /opt/app
+RUN npm install --only=production
+EXPOSE 80
+ENTRYPOINT ["/usr/local/bin/node", "healthcheck.js"]
diff --git a/healthcheck-container/get-status.js b/healthcheck-container/get-status.js
new file mode 100644
index 0000000..2ed1d3d
--- /dev/null
+++ b/healthcheck-container/get-status.js
@@ -0,0 +1,109 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/*
+ * Query Kubernetes for status of deployments and extract readiness information
+ */
+
+const fs = require('fs');
+const request = require('request');
+
+const K8S_CREDS = '/var/run/secrets/kubernetes.io/serviceaccount';
+const K8S_API = 'https://kubernetes.default.svc.cluster.local/'; // Full name to match cert for TLS
+const K8S_PATH = 'apis/apps/v1beta2/namespaces/';
+
+//Get token and CA cert
+const ca = fs.readFileSync(K8S_CREDS + '/ca.crt');
+const token = fs.readFileSync(K8S_CREDS + '/token');
+
+const summarizeDeploymentList = function(list) {
+ // list is a DeploymentList object returned by k8s
+ // Individual deployments are in the array 'items'
+
+ let ret =
+ {
+ type: "summary",
+ count: 0,
+ ready: 0,
+ items: []
+ };
+
+ // Extract readiness information
+ for (let deployment of list.items) {
+ ret.items.push(
+ {
+ name: deployment.metadata.name,
+ ready: deployment.status.readyReplicas || 0,
+ unavailable: deployment.status.unavailableReplicas || 0
+ }
+ );
+ ret.count ++;
+ ret.ready = ret.ready + (deployment.status.readyReplicas || 0);
+ }
+
+ return ret;
+};
+
+const summarizeDeployment = function(deployment) {
+ // deployment is a Deployment object returned by k8s
+ // we make it look enough like a DeploymentList object to
+ // satisfy summarizeDeploymentList
+ return summarizeDeploymentList({items: [deployment]});
+};
+
+const queryKubernetes = function(path, callback) {
+ // Make request to Kubernetes
+
+ const options = {
+ url: K8S_API + path,
+ ca : ca,
+ headers: {
+ Authorization: 'bearer ' + token
+ },
+ json: true
+ };
+ console.log ("request url: " + options.url);
+ request(options, function(error, res, body) {
+ console.log ("status: " + (res && res.statusCode) ? res.statusCode : "NONE");
+ if (error) {
+ console.log("error: " + error);
+ }
+ callback(error, res, body);
+ });
+};
+
+const getStatus = function(path, extract, callback) {
+ // Get info from k8s and extract readiness info
+ queryKubernetes(path, function(error, res, body) {
+ let ret = body;
+ if (!error && res && res.statusCode === 200) {
+ ret = extract(body);
+ }
+ callback (error, res, ret);
+ });
+};
+
+exports.getStatusNamespace = function (namespace, callback) {
+ // Get readiness information for all deployments in namespace
+ const path = K8S_PATH + namespace + '/deployments';
+ getStatus(path, summarizeDeploymentList, callback);
+};
+
+exports.getStatusSingle = function (namespace, deployment, callback) {
+ // Get readiness information for a single deployment
+ const path = K8S_PATH + namespace + '/deployments/' + deployment;
+ getStatus(path, summarizeDeployment, callback);
+}; \ No newline at end of file
diff --git a/healthcheck-container/healthcheck.js b/healthcheck-container/healthcheck.js
new file mode 100644
index 0000000..7555032
--- /dev/null
+++ b/healthcheck-container/healthcheck.js
@@ -0,0 +1,80 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+//Expect ONAP and DCAE namespaces and Helm "release" name to be passed via environment variables
+//
+const ONAP_NS = process.env.ONAP_NAMESPACE || 'default';
+const DCAE_NS = process.env.DCAE_NAMESPACE || 'default';
+const HELM_REL = process.env.HELM_RELEASE || '';
+
+const HEALTHY = 200;
+const UNHEALTHY = 500;
+const UNKNOWN = 503;
+
+const status = require('./get-status');
+const http = require('http');
+
+const isHealthy = function(summary) {
+ // Current healthiness criterion is simple--all deployments are ready
+ return summary.count && summary.ready && summary.count === summary.ready;
+};
+
+const checkHealth = function (callback) {
+ // Makes queries to Kubernetes and checks results
+ // If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500)
+ // If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503)
+ // If we get responses from k8s and all deployments are ready, health status is HEALTHY (200)
+ // This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking
+ status.getStatusNamespace(DCAE_NS, function(err, res, body) {
+ let ret = {status : UNKNOWN, body: [body]};
+ if (err) {
+ callback(ret);
+ }
+ else if (body.type && body.type === 'summary') {
+ if (isHealthy(body)) {
+ // All the DCAE components report healthy -- check Cloudify Manager
+ let cmDeployment = 'dcae-cloudify-manager';
+ if (HELM_REL.length > 0) {
+ cmDeployment = HELM_REL + '-' + cmDeployment;
+ }
+ status.getStatusSingle(ONAP_NS, cmDeployment, function (err, res, body){
+ ret.body.push(body);
+ if (err) {
+ callback(ret);
+ }
+ if (body.type && body.type === 'summary') {
+ ret.status = isHealthy(body) ? HEALTHY : UNHEALTHY;
+ }
+ callback(ret);
+ });
+ }
+ else {
+ callback(ret);
+ }
+ }
+ });
+};
+
+// Simple HTTP server--any incoming request triggers a health check
+const server = http.createServer(function(req, res) {
+ checkHealth(function(ret) {
+ console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret));
+ res.statusCode = ret.status;
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify(ret.body || {}), 'utf8');
+ });
+});
+server.listen(80);
diff --git a/healthcheck-container/package.json b/healthcheck-container/package.json
new file mode 100644
index 0000000..2a08bdd
--- /dev/null
+++ b/healthcheck-container/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "k8s-healthcheck",
+ "description": "DCAE healthcheck server",
+ "version": "1.0.0",
+ "main": "healthcheck.js",
+ "dependencies": {
+ "request": "2.85.0"
+ },
+ "author": "author",
+ "license": "(Apache-2.0)"
+}
diff --git a/healthcheck-container/pom.xml b/healthcheck-container/pom.xml
new file mode 100644
index 0000000..dea3c48
--- /dev/null
+++ b/healthcheck-container/pom.xml
@@ -0,0 +1,172 @@
+<?xml version="1.0"?>
+<!--
+================================================================================
+Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+================================================================================
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+============LICENSE_END=========================================================
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.onap.dcaegen2.deployments</groupId>
+ <artifactId>deployments</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </parent>
+ <groupId>org.onap.dcaegen2.deployments</groupId>
+ <artifactId>healthcheck-container</artifactId>
+ <name>dcaegen2-deployments-healthcheck-container</name>
+ <version>1.0.0</version>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <sonar.skip>true</sonar.skip>
+ <sonar.sources>.</sonar.sources>
+ <!-- customize the SONARQUBE URL -->
+ <!-- sonar.host.url>http://localhost:9000</sonar.host.url -->
+ <!-- below are language dependent -->
+ <!-- for Python -->
+ <sonar.language>py</sonar.language>
+ <sonar.pluginName>Python</sonar.pluginName>
+ <sonar.inclusions>**/*.py</sonar.inclusions>
+ <!-- for JavaScaript -->
+ <!--
+ <sonar.language>js</sonar.language>
+ <sonar.pluginName>JS</sonar.pluginName>
+ <sonar.inclusions>**/*.js</sonar.inclusions>
+ -->
+ </properties>
+ <build>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <plugins>
+ <!-- plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4.1</version>
+ <configuration>
+ <descriptors>
+ <descriptor>assembly/dep.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin -->
+ <!-- now we configure custom action (calling a script) at various lifecycle phases -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <id>clean phase script</id>
+ <phase>clean</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>clean</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>generate-sources script</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>generate-sources</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>compile script</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>compile</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>package script</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>package</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>test script</id>
+ <phase>test</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>test</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>install script</id>
+ <phase>install</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>install</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>deploy script</id>
+ <phase>deploy</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <arguments>
+ <argument>${project.artifactId}</argument>
+ <argument>deploy</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh
index 9897d5e..acada60 100755
--- a/mvn-phase-script.sh
+++ b/mvn-phase-script.sh
@@ -83,7 +83,7 @@ deploy)
upload_files_of_extension sh
build_and_push_docker
;;
- k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container)
+ k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container|healthcheck-container)
build_and_push_docker
;;
scripts|cloud_init|heat)
diff --git a/pom.xml b/pom.xml
index ce585ba..ae1f1b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,6 +44,7 @@ limitations under the License.
<module>cm-container</module>
<module>k8s-bootstrap-container</module>
<module>tca-cdap-container</module>
+ <module>healthcheck-container</module>
</modules>
<properties>