Add aai-resources healthcheck based on Cassandra DB Healthcheck

Issue-ID: AAI-3528

Signed-off-by: Rommel Pawar <rommel.pawar@bell.ca>
Change-Id: I21022bd81c5cd9eed2dd841f96812855adebe528

4 files changed, 345 insertions, 91 deletions

diff --git a/aai-resources/src/main/java/org/onap/aai/ResourcesApp.java b/aai-resources/src/main/java/org/onap/aai/ResourcesApp.java
index 6377af8..419fee5 100644
--- a/aai-resources/src/main/java/org/onap/aai/ResourcesApp.java
+++ b/aai-resources/src/main/java/org/onap/aai/ResourcesApp.java
@@ -1,4 +1,4 @@
-/**
+/*
  * ============LICENSE_START=======================================================
  * org.onap.aai
  * ================================================================================
@@ -17,8 +17,11 @@
  * limitations under the License.
  * ============LICENSE_END=========================================================
  */
+
 package org.onap.aai;
 
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.onap.aai.aailog.logs.AaiDebugLog;
 import org.onap.aai.config.SpringContextAware;
@@ -32,7 +35,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
@@ -40,9 +42,6 @@ import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfigurat
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.core.env.Environment;
 
-import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
-
 @SpringBootApplication(
 	exclude = {
 		DataSourceAutoConfiguration.class,
@@ -87,8 +86,8 @@ public class ResourcesApp {
 	
 	@Autowired
 	private SpringContextAware loaderFactory;
-	
-	
+
+
 	@PostConstruct
 	private void init() throws AAIException {
 		System.setProperty("org.onap.aai.serverStarted", "false");
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/util/EchoResponse.java b/aai-resources/src/main/java/org/onap/aai/rest/util/EchoResponse.java
index b72d113..8b5eef6 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/util/EchoResponse.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/util/EchoResponse.java
@@ -1,4 +1,4 @@
-/**
+/*
  * ============LICENSE_START=======================================================
  * org.onap.aai
  * ================================================================================
@@ -8,7 +8,7 @@
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,12 +17,11 @@
  * limitations under the License.
  * ============LICENSE_END=========================================================
  */
-package org.onap.aai.rest.util;
 
-import org.onap.aai.exceptions.AAIException;
-import org.onap.aai.logging.ErrorLogHelper;
-import org.onap.aai.restcore.RESTAPI;
+package org.onap.aai.rest.util;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
@@ -33,88 +32,123 @@ import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
-import java.util.ArrayList;
-import java.util.HashMap;
+import org.apache.commons.lang3.BooleanUtils;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.restcore.RESTAPI;
+import org.onap.aai.tasks.AaiGraphChecker;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
 
 /**
  * The Class EchoResponse.
  */
+@Component
 @Path("/util")
 public class EchoResponse extends RESTAPI {
-	
-	protected static String authPolicyFunctionName = "util";
-		
-	public static final String echoPath = "/util/echo";
-
-	/**
-	 * Simple health-check API that echos back the X-FromAppId and X-TransactionId to clients.
-	 * If there is a query string, a transaction gets logged into hbase, proving the application is connected to the data store.
-	 * If there is no query string, no transacction logging is done to hbase.
-	 *
-	 * @param headers the headers
-	 * @param req the req
-	 * @param myAction if exists will cause transaction to be logged to hbase
-	 * @return the response
-	 */
-	@GET
-	@Produces( { MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
-	@Path("/echo")
-	public Response echoResult(@Context HttpHeaders headers, @Context HttpServletRequest req,
-			@QueryParam("action") String myAction) {
-		Response response = null;
-		
-		AAIException ex = null;
-		String fromAppId = null;
-		String transId = null;
-		
-		try { 
-			fromAppId = getFromAppId(headers );
-			transId = getTransId(headers);
-		} catch (AAIException e) { 
-			ArrayList<String> templateVars = new ArrayList<String>();
-			templateVars.add("PUT uebProvider");
-			templateVars.add("addTopic");
-			return Response
-					.status(e.getErrorObject().getHTTPResponseCode())
-					.entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), e, templateVars))
-					.build();
-		}
-		
-		try {
-			
-			HashMap<AAIException, ArrayList<String>> exceptionList = new HashMap<AAIException, ArrayList<String>>();
-					
-			ArrayList<String> templateVars = new ArrayList<String>();
-			templateVars.add(fromAppId);
-			templateVars.add(transId);
-		
-			exceptionList.put(new AAIException("AAI_0002", "OK"), templateVars);
-				
-			response = Response.status(Status.OK)
-					.entity(ErrorLogHelper.getRESTAPIInfoResponse(
-							headers.getAcceptableMediaTypes(), exceptionList))
-							.build();
-			
-		} catch (Exception e) {
-			ex = new AAIException("AAI_4000", e);
-			ArrayList<String> templateVars = new ArrayList<String>();
-			templateVars.add(Action.GET.name());
-			templateVars.add(fromAppId +" "+transId);
-
-			response = Response
-					.status(Status.INTERNAL_SERVER_ERROR)
-					.entity(ErrorLogHelper.getRESTAPIErrorResponse(
-							headers.getAcceptableMediaTypes(), ex,
-							templateVars)).build();
-
-		} finally {
-			if (ex != null) {
-				ErrorLogHelper.logException(ex);
-			}
-
-		}
-		
-		return response;
-	}
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(EchoResponse.class);
+
+    private static final String CHECK_DB_STATUS_ACTION = "checkDB";
+
+    private static final String CHECK_DB_STATUS_NOW_ACTION = "checkDBNow";
+
+    private final AaiGraphChecker aaiGraphChecker;
+
+    public EchoResponse(AaiGraphChecker aaiGraphChecker) {
+        this.aaiGraphChecker = aaiGraphChecker;
+    }
+
+    /**
+     * Simple health-check API that echos back the X-FromAppId and X-TransactionId to clients.
+     * If there is a query string, a transaction gets logged into hbase, proving the application is connected to the
+     * data store.
+     * If there is no query string, no transacction logging is done to hbase.
+     *
+     * @param headers  the headers
+     * @param req      the req
+     * @param myAction if exists will cause transaction to be logged to hbase
+     * @return the response
+     */
+    @GET
+    @Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
+    @Path("/echo")
+    public Response echoResult(
+        @Context HttpHeaders headers, @Context HttpServletRequest req, @QueryParam("action") String myAction) {
+
+        String fromAppId;
+        String transId;
+        try {
+            fromAppId = getFromAppId(headers);
+            transId = getTransId(headers);
+        } catch (AAIException aaiException) {
+            ArrayList<String> templateVars = new ArrayList<>();
+            templateVars.add("PUT uebProvider");
+            templateVars.add("addTopic");
+            LOGGER.error("Error while getting ids", aaiException);
+            return generateFailureResponse(headers, templateVars, aaiException);
+        }
+
+        ArrayList<String> templateVars = new ArrayList<>();
+        templateVars.add(fromAppId);
+        templateVars.add(transId);
+
+        try {
+            if (CHECK_DB_STATUS_ACTION.equalsIgnoreCase(myAction) ||
+                CHECK_DB_STATUS_NOW_ACTION.equalsIgnoreCase(myAction)) {
+                validateDBStatus(myAction);
+            }
+            return generateSuccessResponse(headers, templateVars);
+
+        } catch (AAIException aaiException) {
+            LOGGER.error("Error while processing echo request ", aaiException);
+            return generateFailureResponse(headers, templateVars, aaiException);
+        } catch (Exception exception) {
+            AAIException aaiException = new AAIException("AAI_4000", exception);
+            LOGGER.error("Error while generating echo response", exception);
+            return generateFailureResponse(headers, templateVars, aaiException);
+        }
+    }
+
+    /**
+     * Validates if Janus Graph can process request using AAIGraphChecker.
+     *
+     * @param action        expected input values 'checkDB' 'checkDBNow'
+     * @throws AAIException exception thrown if DB is not available
+     */
+    private void validateDBStatus(String action) throws AAIException {
+
+        Boolean dbAvailable = null;
+        if (CHECK_DB_STATUS_ACTION.equalsIgnoreCase(action)) {
+            dbAvailable = aaiGraphChecker.isAaiGraphDbAvailable(AaiGraphChecker.CheckerType.CACHED);
+        } else if (CHECK_DB_STATUS_NOW_ACTION.equalsIgnoreCase(action)) {
+            dbAvailable = aaiGraphChecker.isAaiGraphDbAvailable(AaiGraphChecker.CheckerType.ACTUAL);
+        } else {
+            LOGGER.error("Invalid check db action specified to generate echo response: '{}'", action);
+        }
+
+        if (BooleanUtils.isFalse(dbAvailable)) {
+            throw new AAIException("AAI_5105", "Error establishing a database connection");
+        }
+
+    }
+
+    private Response generateSuccessResponse(HttpHeaders headers, ArrayList<String> templateVariables) {
+        HashMap<AAIException, ArrayList<String>> exceptionList = new HashMap<>();
+        exceptionList.put(new AAIException("AAI_0002", "OK"), templateVariables);
+        return Response.status(Status.OK)
+            .entity(
+                ErrorLogHelper.getRESTAPIInfoResponse(headers.getAcceptableMediaTypes(), exceptionList)).build();
+    }
+
+    private Response generateFailureResponse(HttpHeaders headers, ArrayList<String> templateVariables,
+        AAIException aaiException) {
+        return Response.status(aaiException.getErrorObject().getHTTPResponseCode())
+            .entity(
+                ErrorLogHelper.getRESTAPIErrorResponseWithLogging(
+                    headers.getAcceptableMediaTypes(), aaiException, templateVariables))
+            .build();
+    }
 
 }
diff --git a/aai-resources/src/main/java/org/onap/aai/tasks/AaiGraphChecker.java b/aai-resources/src/main/java/org/onap/aai/tasks/AaiGraphChecker.java
new file mode 100644
index 0000000..d1ea478
--- /dev/null
+++ b/aai-resources/src/main/java/org/onap/aai/tasks/AaiGraphChecker.java
@@ -0,0 +1,208 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2022 Bell Canada
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.aai.tasks;
+
+import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.Timer;
+import java.util.TimerTask;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.Validate;
+import org.janusgraph.core.JanusGraphException;
+import org.janusgraph.core.JanusGraphTransaction;
+import org.janusgraph.core.JanusGraphVertex;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.util.AAIConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+/**
+ * Singleton class responsible to check that AAI service is able to connect to its back-end database.
+ * The check can run as a scheduled task or on demand.
+ */
+@Component
+@Scope(ConfigurableBeanFactory.SCOPE_SINGLETON)
+public class AaiGraphChecker extends TimerTask {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(AaiGraphChecker.class);
+
+    // Default indicator to enable or disable scheduled task
+    private static final String DEFAULT_SCHEDULE_ENABLED_VALUE = "false";
+    // Default delay, in seconds, before the scheduled task is started, if enabled
+    private static final String DEFAULT_SCHEDULE_DELAY_VALUE = "5";
+    // Default period, in seconds, between two consecutive executions of the scheduled task, if enabled
+    private static final String DEFAULT_SCHEDULE_PERIOD_VALUE = "60";
+
+    // Database availability cached indicator
+    private volatile Boolean isAaiGraphDbAvailableCache = null;
+
+    private Timer timer = null;
+
+    /**
+     * Enumeration of check type that can be made.
+     */
+    public enum CheckerType {
+        ACTUAL,
+        CACHED
+    }
+
+    private AaiGraphChecker() {
+    }
+
+    @PostConstruct
+    private void setupTimer() {
+
+        boolean scheduleEnabled =
+                Boolean.parseBoolean(
+                        getConfigurationValueOrDefault(
+                                "aai.graph.checker.task.enabled", DEFAULT_SCHEDULE_ENABLED_VALUE));
+        long scheduleDelay =
+                Long.parseLong(
+                        getConfigurationValueOrDefault(
+                                "aai.graph.checker.task.delay", DEFAULT_SCHEDULE_DELAY_VALUE));
+        long schedulePeriod =
+                Long.parseLong(
+                        getConfigurationValueOrDefault(
+                                "aai.graph.checker.task.period", DEFAULT_SCHEDULE_PERIOD_VALUE));
+        LOGGER.debug(
+                "Setting up AaiGraphChecker with scheduleEnabled={}, scheduleDelay={}, schedulePeriod={} ",
+                scheduleEnabled, scheduleDelay, schedulePeriod);
+
+        if (scheduleEnabled) {
+            timer = new Timer();
+            timer.schedule(this, scheduleDelay * 1000, schedulePeriod * 1000);
+        }
+    }
+
+    @PreDestroy
+    private void tearDownTimer() {
+        LOGGER.debug("Tear down AaiGraphChecker");
+        if (timer != null) {
+            timer.cancel();
+            timer = null;
+        }
+    }
+
+    @Override
+    public void run() {
+        isAaiGraphDbAvailable(CheckerType.ACTUAL);
+    }
+
+    /**
+     * Clear database availability cached indicator.
+     */
+    public void clearDbAvailabilityCachedIndicator() {
+        isAaiGraphDbAvailableCache = null;
+    }
+
+    /**
+     * Indicate if AAI Graph database is available either from actual db connection or from cached property state.
+     * @param checkerType the type of check to be made (actual or cached). Null is not supported.
+     * @return
+     *      <li>true, if database is available</li>
+     *      <li>false, if database is NOT available</li>
+     *      <li>null, if database availability can not be determined</li>
+     */
+    public Boolean isAaiGraphDbAvailable(CheckerType checkerType) {
+        Validate.notNull(checkerType);
+        if (CheckerType.ACTUAL.equals(checkerType)) {
+            isAaiGraphDbAvailableCache = isAaiGraphDbAvailableActual();
+        }
+        logDbState(checkerType);
+        return isAaiGraphDbAvailableCache;
+    }
+
+    private Boolean isAaiGraphDbAvailableActual() {
+        Boolean dbAvailable;
+        JanusGraphTransaction transaction = null;
+        try {
+            transaction = AAIGraph.getInstance().getGraph().newTransaction();
+            final Iterator<JanusGraphVertex> vertexIterator = transaction.query().limit(1).vertices().iterator();
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug("Number of vertices retrieved while checking db: {}", Iterators.size(vertexIterator));
+            }
+            vertexIterator.hasNext();
+            LOGGER.debug("Actual database availability is true");
+            dbAvailable = Boolean.TRUE;
+        } catch (JanusGraphException e) {
+            String message = "Actual database availability is false (after JanusGraph exception)";
+            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
+            LOGGER.error(message, e);
+            dbAvailable = Boolean.FALSE;
+        } catch (Error e) {
+            // Following error occurs when aai resources is starting:
+            // - UnsatisfiedLinkError (for org.onap.aai.dbmap.AAIGraph$Helper instantiation)
+            // Following errors are raised when aai resources is starting and cassandra is not running:
+            // - ExceptionInInitializerError
+            // - NoClassDefFoundError (definition for org.onap.aai.dbmap.AAIGraph$Helper is not found)
+            String message = "Actual database availability is false (after error)";
+            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
+            LOGGER.error(message, e);
+            dbAvailable = Boolean.FALSE;
+        } catch (Exception e) {
+            String message = "Actual database availability can not be determined";
+            ErrorLogHelper.logError("500", message + ": " + e.getMessage());
+            LOGGER.error(message, e);
+            dbAvailable = null;
+        } finally {
+            if (transaction != null && !transaction.isClosed()) {
+                //check if transaction is open then close instead of flag
+                try {
+                    transaction.rollback();
+                } catch (Exception e) {
+                    String message = "Exception occurred while closing transaction";
+                    LOGGER.error(message, e);
+                    ErrorLogHelper.logError("500", message + ": " + e.getMessage());
+                }
+            }
+        }
+        return dbAvailable;
+    }
+
+    private void logDbState(CheckerType type) {
+        if (BooleanUtils.isTrue(isAaiGraphDbAvailableCache)) {
+            LOGGER.debug("Database is available from {} check.", type);
+        } else if (BooleanUtils.isFalse(isAaiGraphDbAvailableCache)) {
+            LOGGER.error("Database is NOT available from {} check.", type);
+        } else {
+            LOGGER.error("Database availability is UNKNOWN from {} check.", type);
+        }
+    }
+
+    private String getConfigurationValueOrDefault(String property, String defaultValue) {
+        String result;
+        try {
+            result = AAIConfig.get(property);
+        } catch (AAIException e) {
+            LOGGER.error(
+                    "Unable to get defined configuration value for '{}' property, then default '{}' value is used",
+                    property, defaultValue);
+            result = defaultValue;
+        }
+        return result;
+    }
+
+}
diff --git a/aai-resources/src/main/resources/etc/appprops/aaiconfig.properties b/aai-resources/src/main/resources/etc/appprops/aaiconfig.properties
index 42fbbfc..4bcb2f3 100644
--- a/aai-resources/src/main/resources/etc/appprops/aaiconfig.properties
+++ b/aai-resources/src/main/resources/etc/appprops/aaiconfig.properties
@@ -157,4 +157,17 @@ aai.implied.delete.log.enabled=false
 aai.implied.delete.whitelist.sdnc='vce'
 aai.implied.delete.whitelist.ro='vserver'
 
-property.null.validation.enabled=true
\ No newline at end of file
+property.null.validation.enabled=true
+
+#
+# AAI Graph DB checker task
+#
+
+# Indicator to enable or disable scheduled task (true/false)
+aai.graph.checker.task.enabled=false
+
+# Delay, in seconds, before the scheduled task is started, if enabled
+aai.graph.checker.task.delay=5
+
+# Period, in seconds, between two consecutive executions of the scheduled task, if enabled
+aai.graph.checker.task.period=10