Add healthcheck container for OOM

Change-Id: Ie4719a0e4705901fd9d0fa3504696fbefc6c704a
Issue-ID: DCAEGEN2-461
Signed-off-by: Jack Lucas <jflucas@research.att.com>

7 files changed, 382 insertions, 1 deletions

diff --git a/healthcheck-container/Dockerfile b/healthcheck-container/Dockerfile
new file mode 100644
index 0000000..d1b4231
--- /dev/null
+++ b/healthcheck-container/Dockerfile
@@ -0,0 +1,8 @@
+FROM node:8.11.1
+RUN mkdir -p /opt/app
+COPY *.js /opt/app/
+COPY package.json /opt/app/
+WORKDIR /opt/app
+RUN npm install --only=production
+EXPOSE 80
+ENTRYPOINT ["/usr/local/bin/node", "healthcheck.js"]
diff --git a/healthcheck-container/get-status.js b/healthcheck-container/get-status.js
new file mode 100644
index 0000000..2ed1d3d
--- /dev/null
+++ b/healthcheck-container/get-status.js
@@ -0,0 +1,109 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+/*
+ * Query Kubernetes for status of deployments and extract readiness information
+ */
+
+const fs = require('fs');
+const request = require('request');
+
+const K8S_CREDS = '/var/run/secrets/kubernetes.io/serviceaccount';
+const K8S_API = 'https://kubernetes.default.svc.cluster.local/';	// Full name to match cert for TLS
+const K8S_PATH = 'apis/apps/v1beta2/namespaces/';
+
+//Get token and CA cert
+const ca = fs.readFileSync(K8S_CREDS + '/ca.crt');
+const token = fs.readFileSync(K8S_CREDS + '/token');
+
+const summarizeDeploymentList = function(list) {
+	// list is a DeploymentList object returned by k8s
+	// Individual deployments are in the array 'items'
+	
+	let ret = 
+	{
+		type: "summary",
+		count: 0,
+		ready: 0,
+		items: []
+	};
+	
+	// Extract readiness information
+	for (let deployment of list.items) {
+		ret.items.push(
+			{
+				name: deployment.metadata.name,
+				ready: deployment.status.readyReplicas || 0,
+				unavailable: deployment.status.unavailableReplicas || 0
+			}
+		);
+		ret.count ++;
+		ret.ready = ret.ready + (deployment.status.readyReplicas || 0);
+	}
+	
+	return ret;
+};
+
+const summarizeDeployment = function(deployment) {
+	// deployment is a Deployment object returned by k8s
+	// we make it look enough like a DeploymentList object to
+	// satisfy summarizeDeploymentList
+	return summarizeDeploymentList({items: [deployment]});
+};
+
+const queryKubernetes = function(path, callback) {
+	// Make request to Kubernetes
+	
+	const options = {
+		url: K8S_API + path,
+		ca : ca,
+		headers: {
+			Authorization: 'bearer ' + token
+		},
+		json: true
+	};
+	console.log ("request url: " + options.url);
+	request(options, function(error, res, body) {
+		console.log ("status: " + (res && res.statusCode) ? res.statusCode : "NONE");
+		if (error) {
+			console.log("error: " + error);
+		}
+		callback(error, res, body);
+	});
+};
+
+const getStatus = function(path, extract, callback) {
+	// Get info from k8s and extract readiness info
+	queryKubernetes(path, function(error, res, body) {
+		let ret = body;
+		if (!error && res && res.statusCode === 200) {
+			ret = extract(body);
+		}
+		callback (error, res, ret);
+	});
+};
+
+exports.getStatusNamespace = function (namespace, callback) {
+	// Get readiness information for all deployments in namespace
+	const path = K8S_PATH + namespace + '/deployments';
+	getStatus(path, summarizeDeploymentList, callback);
+};
+
+exports.getStatusSingle = function (namespace, deployment, callback) {
+	// Get readiness information for a single deployment
+	const path = K8S_PATH + namespace + '/deployments/' + deployment;
+	getStatus(path, summarizeDeployment, callback);
+};
\ No newline at end of file
diff --git a/healthcheck-container/healthcheck.js b/healthcheck-container/healthcheck.js
new file mode 100644
index 0000000..7555032
--- /dev/null
+++ b/healthcheck-container/healthcheck.js
@@ -0,0 +1,80 @@
+/*
+Copyright(c) 2018 AT&T Intellectual Property. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+*/
+
+//Expect ONAP and DCAE namespaces and Helm "release" name to be passed via environment variables
+// 
+const ONAP_NS = process.env.ONAP_NAMESPACE || 'default';
+const DCAE_NS = process.env.DCAE_NAMESPACE || 'default';
+const HELM_REL = process.env.HELM_RELEASE || '';
+
+const HEALTHY = 200;
+const UNHEALTHY = 500;
+const UNKNOWN = 503;
+
+const status = require('./get-status');
+const http = require('http');
+
+const isHealthy = function(summary) {
+	// Current healthiness criterion is simple--all deployments are ready
+	return summary.count && summary.ready && summary.count === summary.ready;
+};
+
+const checkHealth = function (callback) {
+	// Makes queries to Kubernetes and checks results
+	// If we encounter some kind of error contacting k8s (or other), health status is UNKNOWN (500)
+	// If we get responses from k8s but don't find all deployments ready, health status is UNHEALTHY (503)
+	// If we get responses from k8s and all deployments are ready, health status is HEALTHY (200)
+	// This could be a lot more nuanced, but what's here should be sufficient for R2 OOM healthchecking
+	status.getStatusNamespace(DCAE_NS, function(err, res, body) {
+		let ret = {status : UNKNOWN, body: [body]};
+		if (err) {
+			callback(ret);
+		}
+		else if (body.type && body.type === 'summary') {
+			if (isHealthy(body)) {
+				// All the DCAE components report healthy -- check Cloudify Manager
+				let cmDeployment = 'dcae-cloudify-manager';
+				if (HELM_REL.length > 0) {
+					cmDeployment = HELM_REL + '-' + cmDeployment;
+				}
+				status.getStatusSingle(ONAP_NS, cmDeployment, function (err, res, body){
+					ret.body.push(body);
+					if (err) {
+						callback(ret);
+					}
+					if (body.type && body.type === 'summary') {
+						ret.status = isHealthy(body) ? HEALTHY : UNHEALTHY;
+					}
+					callback(ret);
+				});
+			}
+			else {
+				callback(ret);
+			}
+		}
+	});
+};
+
+// Simple HTTP server--any incoming request triggers a health check
+const server = http.createServer(function(req, res) {
+	checkHealth(function(ret) {
+		console.log ((new Date()).toISOString() + ": " + JSON.stringify(ret));
+		res.statusCode = ret.status;
+		res.setHeader('Content-Type', 'application/json');
+		res.end(JSON.stringify(ret.body || {}), 'utf8');
+	});
+});
+server.listen(80);
diff --git a/healthcheck-container/package.json b/healthcheck-container/package.json
new file mode 100644
index 0000000..2a08bdd
--- /dev/null
+++ b/healthcheck-container/package.json
@@ -0,0 +1,11 @@
+{
+  "name": "k8s-healthcheck",
+  "description": "DCAE healthcheck server",
+  "version": "1.0.0",
+  "main": "healthcheck.js",
+  "dependencies": {
+    "request": "2.85.0"
+  },
+  "author": "author",
+  "license": "(Apache-2.0)"
+}
diff --git a/healthcheck-container/pom.xml b/healthcheck-container/pom.xml
new file mode 100644
index 0000000..dea3c48
--- /dev/null
+++ b/healthcheck-container/pom.xml
@@ -0,0 +1,172 @@
+<?xml version="1.0"?>
+<!--
+================================================================================
+Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
+================================================================================
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+============LICENSE_END=========================================================
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.onap.dcaegen2.deployments</groupId>
+    <artifactId>deployments</artifactId>
+    <version>1.2.0-SNAPSHOT</version>
+  </parent>
+  <groupId>org.onap.dcaegen2.deployments</groupId>
+  <artifactId>healthcheck-container</artifactId>
+  <name>dcaegen2-deployments-healthcheck-container</name>
+  <version>1.0.0</version>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <sonar.skip>true</sonar.skip>
+    <sonar.sources>.</sonar.sources>
+    <!-- customize the SONARQUBE URL -->
+    <!-- sonar.host.url>http://localhost:9000</sonar.host.url -->
+    <!-- below are language dependent -->
+    <!-- for Python -->
+    <sonar.language>py</sonar.language>
+    <sonar.pluginName>Python</sonar.pluginName>
+    <sonar.inclusions>**/*.py</sonar.inclusions>
+    <!-- for JavaScaript -->
+    <!--
+    <sonar.language>js</sonar.language>
+    <sonar.pluginName>JS</sonar.pluginName>
+    <sonar.inclusions>**/*.js</sonar.inclusions>
+    -->
+  </properties>
+  <build>
+    <finalName>${project.artifactId}-${project.version}</finalName>
+    <plugins>
+      <!-- plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>2.4.1</version>
+        <configuration>
+          <descriptors>
+            <descriptor>assembly/dep.xml</descriptor>
+          </descriptors>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin -->
+      <!-- now we configure custom action (calling a script) at various lifecycle phases -->
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <id>clean phase script</id>
+            <phase>clean</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>clean</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>generate-sources script</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>generate-sources</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>compile script</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>compile</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>package script</id>
+            <phase>package</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>package</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>test script</id>
+            <phase>test</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>test</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>install script</id>
+            <phase>install</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>install</argument>
+              </arguments>
+            </configuration>
+          </execution>
+          <execution>
+            <id>deploy script</id>
+            <phase>deploy</phase>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <configuration>
+              <arguments>
+                <argument>${project.artifactId}</argument>
+                <argument>deploy</argument>
+              </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh
index 9897d5e..acada60 100755
--- a/mvn-phase-script.sh
+++ b/mvn-phase-script.sh
@@ -83,7 +83,7 @@ deploy)
     upload_files_of_extension sh
     build_and_push_docker
     ;;
-  k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container)
+  k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container|healthcheck-container)
     build_and_push_docker
     ;;
   scripts|cloud_init|heat)
diff --git a/pom.xml b/pom.xml
index ce585ba..ae1f1b9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,6 +44,7 @@ limitations under the License.
      <module>cm-container</module>
      <module>k8s-bootstrap-container</module>
      <module>tca-cdap-container</module>
+     <module>healthcheck-container</module>
   </modules>
 
   <properties>