aboutsummaryrefslogtreecommitdiffstats
path: root/integrity-monitor/src
diff options
context:
space:
mode:
Diffstat (limited to 'integrity-monitor/src')
-rw-r--r--integrity-monitor/src/main/java/org/onap/policy/common/im/AllSeemsWellException.java45
-rw-r--r--integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitor.java1776
-rw-r--r--integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitorProperties.java7
-rw-r--r--integrity-monitor/src/test/java/org/onap/policy/common/im/test/AllSeemsWellTest.java265
-rw-r--r--integrity-monitor/src/test/java/org/onap/policy/common/im/test/IntegrityMonitorTest.java528
-rw-r--r--integrity-monitor/src/test/resources/log4j.properties54
-rw-r--r--integrity-monitor/src/test/resources/logback-test.xml47
-rw-r--r--integrity-monitor/src/test/resources/logback.xml209
8 files changed, 1666 insertions, 1265 deletions
diff --git a/integrity-monitor/src/main/java/org/onap/policy/common/im/AllSeemsWellException.java b/integrity-monitor/src/main/java/org/onap/policy/common/im/AllSeemsWellException.java
new file mode 100644
index 00000000..5878e1d1
--- /dev/null
+++ b/integrity-monitor/src/main/java/org/onap/policy/common/im/AllSeemsWellException.java
@@ -0,0 +1,45 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * Integrity Monitor
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.common.im;
+
+public class AllSeemsWellException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public AllSeemsWellException(){
+ super();
+ }
+
+ public AllSeemsWellException(String msg) {
+ super(msg);
+ }
+
+ public AllSeemsWellException(String msg, Exception cause) {
+ super(msg, cause);
+ }
+
+ public AllSeemsWellException(Exception cause) {
+ super(cause);
+ }
+}
diff --git a/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitor.java b/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitor.java
index 93cee577..5b18b05d 100644
--- a/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitor.java
+++ b/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitor.java
@@ -24,7 +24,9 @@ import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
@@ -37,6 +39,7 @@ import javax.persistence.FlushModeType;
import javax.persistence.LockModeType;
import javax.persistence.Persistence;
import javax.persistence.Query;
+import javax.validation.constraints.NotNull;
import org.onap.policy.common.im.jpa.ForwardProgressEntity;
import org.onap.policy.common.im.jpa.ResourceRegistrationEntity;
@@ -48,110 +51,114 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * IntegrityMonitor
- * Main class for monitoring the integrity of a resource and managing its state. State management follows
- * the X.731 ITU standard.
+ * IntegrityMonitor Main class for monitoring the integrity of a resource and
+ * managing its state. State management follows the X.731 ITU standard.
*/
public class IntegrityMonitor {
- private static final Logger logger = LoggerFactory.getLogger(IntegrityMonitor.class.getName());
-
-
+ private static final Logger logger = LoggerFactory
+ .getLogger(IntegrityMonitor.class.getName());
+
// only allow one instance of IntegrityMonitor
private static IntegrityMonitor instance = null;
-
+
private static String resourceName = null;
boolean alarmExists = false;
-
+
/*
- * Error message that is written by the dependencyCheck() method. It is made available externally
- * through the evaluateSanity() method.
+ * Error message that is written by the dependencyCheck() method. It is made
+ * available externally through the evaluateSanity() method.
*/
private String dependencyCheckErrorMsg = "";
-
-
+
// The entity manager factory for JPA access
private EntityManagerFactory emf;
private EntityManager em;
- // Persistence Unit for JPA
+ // Persistence Unit for JPA
private static final String PERSISTENCE_UNIT = "operationalPU";
-
+
private StateManagement stateManager = null;
-
+
private static final int CYCLE_INTERVAL_MILLIS = 1000;
-
+
+ private FPManager fpManager = null;
+
// The forward progress counter is incremented as the
// process being monitored makes forward progress
private int fpCounter = 0;
private int lastFpCounter = 0;
-
+
// elapsed time since last FP counter check
private long elapsedTime = 0;
-
+
// elapsed time since last test transaction check
private long elapsedTestTransTime = 0;
-
+
// elapsed time since last write Fpc check
private long elapsedWriteFpcTime = 0;
-
- // last dependency health check time. Initialize so that the periodic check starts after 60 seconds.
+
+ // last dependency health check time. Initialize so that the periodic check
+ // starts after 60 seconds.
// This allows time for dependents to come up.
private long lastDependencyCheckTime = System.currentTimeMillis();
-
- // Time of the last state audit. It is initialized at the time of the IM construction
+
+ // Time of the last state audit. It is initialized at the time of the IM
+ // construction
private Date lastStateAuditTime = new Date();
-
- //Interval between state audits in ms. We leave it turned off by default so that it will only
- //be run on the nodes which we want doing the audit. In particular, we only want it to run
- //on the droolspdps
+
+ // Interval between state audits in ms. We leave it turned off by default so
+ // that it will only
+ // be run on the nodes which we want doing the audit. In particular, we only
+ // want it to run
+ // on the droolspdps
private static long stateAuditIntervalMs = 0L;
-
+
// the number of cycles since 'fpCounter' was last changed
private int missedCycles = 0;
-
+
// forward progress monitoring interval
private static int monitorInterval = IntegrityMonitorProperties.DEFAULT_MONITOR_INTERVAL;
- // The number of periods the counter fails to increment before an alarm is raised.
+ // The number of periods the counter fails to increment before an alarm is
+ // raised.
private static int failedCounterThreshold = IntegrityMonitorProperties.DEFAULT_FAILED_COUNTER_THRESHOLD;
// test transaction interval
private static int testTransInterval = IntegrityMonitorProperties.DEFAULT_TEST_INTERVAL;
// write Fpc to DB interval
private static int writeFpcInterval = IntegrityMonitorProperties.DEFAULT_WRITE_FPC_INTERVAL;
+ // check the health of dependencies
+ private static int checkDependencyInterval = IntegrityMonitorProperties.DEFAULT_CHECK_DEPENDENCY_INTERVAL;
- // A lead subsystem will have dependency groups with resource names in the properties file.
+
+ // A lead subsystem will have dependency groups with resource names in the
+ // properties file.
// For non-lead subsystems, the dependency_group property will be absent.
- private static String [] dep_groups = null;
-
+ private static String[] dep_groups = null;
+
private static boolean isUnitTesting = false;
- // can turn on health checking of dependents via jmx test() call by setting this property to true
+ // can turn on health checking of dependents via jmx test() call by setting
+ // this property to true
private static boolean testViaJmx = false;
-
+
private static String jmxFqdn = null;
- // this is the max interval seconds allowed without any forward progress counter updates
+ // this is the max interval seconds allowed without any forward progress
+ // counter updates
private static int maxFpcUpdateInterval = IntegrityMonitorProperties.DEFAULT_MAX_FPC_UPDATE_INTERVAL;
-
+
// Node types
private enum NodeType {
- pdp_xacml,
- pdp_drools,
- pap,
- pap_admin,
- logparser,
- brms_gateway,
- astra_gateway,
- elk_server,
- pypdp
+ pdp_xacml, pdp_drools, pap, pap_admin, logparser, brms_gateway, astra_gateway, elk_server, pypdp
}
-
+
private static String site_name;
private static String node_type;
private Date refreshStateAuditLastRunDate;
- private static long refreshStateAuditIntervalMs = 600000; //run it once per 10 minutes
-
- //lock objects
+ private static long refreshStateAuditIntervalMs = 600000; // run it once per
+ // 10 minutes
+
+ // lock objects
private final Object evaluateSanityLock = new Object();
private final Object fpMonitorCycleLock = new Object();
private final Object dependencyCheckLock = new Object();
@@ -164,26 +171,38 @@ public class IntegrityMonitor {
private final Object refreshStateAuditLock = new Object();
private final Object IMFLUSHLOCK = new Object();
+ private Map<String, String> allSeemsWellMap;
+ private Map<String, String> allNotWellMap;
+
/**
- * Get an instance of IntegrityMonitor for a given resource name. It creates one if it does not exist.
- * Only one instance is allowed to be created per resource name.
- * @param resourceName The resource name of the resource
- * @param properties a set of properties passed in from the resource
+ * Get an instance of IntegrityMonitor for a given resource name. It creates
+ * one if it does not exist. Only one instance is allowed to be created per
+ * resource name.
+ *
+ * @param resourceName
+ * The resource name of the resource
+ * @param properties
+ * a set of properties passed in from the resource
* @return The new instance of IntegrityMonitor
- * @throws Exception if unable to create jmx url or the constructor returns an exception
+ * @throws Exception
+ * if unable to create jmx url or the constructor returns an
+ * exception
*/
- public static IntegrityMonitor getInstance(String resourceName, Properties properties) throws Exception {
- synchronized(getInstanceLock){
- if(logger.isDebugEnabled()){
- logger.debug("getInstance() called - resourceName= {}", resourceName);
- }
- if (resourceName == null || resourceName.isEmpty() || properties == null) {
+ public static IntegrityMonitor getInstance(String resourceName,
+ Properties properties) throws Exception {
+ synchronized (getInstanceLock) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("getInstance() called - resourceName= {}",
+ resourceName);
+ }
+ if (resourceName == null || resourceName.isEmpty()
+ || properties == null) {
logger.error("Error: getIntegrityMonitorInstance() called with invalid input");
return null;
}
if (instance == null) {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("Creating new instance of IntegrityMonitor");
}
instance = new IntegrityMonitor(resourceName, properties);
@@ -191,213 +210,276 @@ public class IntegrityMonitor {
return instance;
}
}
-
- public static IntegrityMonitor getInstance() throws Exception{
- if(logger.isDebugEnabled()){
+
+ public static IntegrityMonitor getInstance() throws Exception {
+ if (logger.isDebugEnabled()) {
logger.debug("getInstance() called");
}
if (instance == null) {
String msg = "No IntegrityMonitor instance exists."
+ " Please use the method IntegrityMonitor.getInstance(String resourceName, Properties properties)";
throw new IntegrityMonitorPropertiesException(msg);
- }else{
+ } else {
return instance;
}
}
-
- public static void deleteInstance(){
- if(logger.isDebugEnabled()){
+
+ /*
+ * This is a facility used by JUnit testing to destroy the IntegrityMonitor
+ * instance before creating a new one. It includes a delay of 2 seconds to
+ * allow the FPManager to fully exit.
+ */
+ public static void deleteInstance() throws IntegrityMonitorException {
+ if (logger.isDebugEnabled()) {
logger.debug("deleteInstance() called");
}
- if(isUnitTesting()){
- instance=null;
+ if (isUnitTesting()) {
+ // Stop the FPManager thread
+
+ if (instance != null && instance.getFPManager() != null) {
+ instance.getFPManager().stopAndExit();
+
+ try {
+ // Make sure it has exited
+ Thread.sleep(2 * CYCLE_INTERVAL_MILLIS);
+ } catch (InterruptedException e) {
+ logger.error("deleteInstance: Interrupted while waiting for FPManaager to fully exit");
+ e.printStackTrace();
+ }
+ if (instance.getFPManager().isAlive()) {
+ logger.error("IntegrityMonitor.deleteInstance() Failed to kill FPManager thread");
+ throw new IntegrityMonitorException(
+ "IntegrityMonitor.deleteInstance() Failed to kill FPManager thread");
+ }
+ instance = null;
+ }
}
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("deleteInstance() exit");
}
}
+
+ private FPManager getFPManager() {
+ return fpManager;
+ }
+
/**
- * IntegrityMonitor constructor. It is invoked from the getInstance() method in
- * this class or from the constructor of a child or sub-class. A class can extend
- * the IntegrityMonitor class if there is a need to override any of the base
- * methods (ex. subsystemTest()). Only one instance is allowed to be created per
- * resource name.
- * @param resourceName The resource name of the resource
- * @param properties a set of properties passed in from the resource
- * @throws Exception if any errors are encountered in the consructor
+ * IntegrityMonitor constructor. It is invoked from the getInstance() method
+ * in this class or from the constructor of a child or sub-class. A class
+ * can extend the IntegrityMonitor class if there is a need to override any
+ * of the base methods (ex. subsystemTest()). Only one instance is allowed
+ * to be created per resource name.
+ *
+ * @param resourceName
+ * The resource name of the resource
+ * @param properties
+ * a set of properties passed in from the resource
+ * @throws Exception
+ * if any errors are encountered in the consructor
*/
- protected IntegrityMonitor(String resourceName, Properties properties) throws Exception {
-
- // singleton check since this constructor can be called from a child or sub-class
+ protected IntegrityMonitor(String resourceName, Properties properties)
+ throws Exception {
+
+ // singleton check since this constructor can be called from a child or
+ // sub-class
if (instance != null) {
String msg = "IM object exists and only one instance allowed";
logger.error("{}", msg);
- throw new IntegrityMonitorException("IntegrityMonitor constructor exception: " + msg);
+ throw new IntegrityMonitorException(
+ "IntegrityMonitor constructor exception: " + msg);
}
instance = this;
-
+
IntegrityMonitor.resourceName = resourceName;
/*
- * Validate that the properties file contains all the needed properties. Throws
- * an IntegrityMonitorPropertiesException
+ * Validate that the properties file contains all the needed properties.
+ * Throws an IntegrityMonitorPropertiesException
*/
validateProperties(properties);
-
+
// construct jmx url
String jmxUrl = getJmxUrl();
-
+
//
// Create the entity manager factory
//
- emf = Persistence.createEntityManagerFactory(PERSISTENCE_UNIT, properties);
+ emf = Persistence.createEntityManagerFactory(PERSISTENCE_UNIT,
+ properties);
//
// Did it get created?
//
if (emf == null) {
- logger.error("Error creating IM entity manager factory with persistence unit: {}",
- PERSISTENCE_UNIT);
- throw new IntegrityMonitorException("Unable to create IM Entity Manager Factory");
+ logger.error(
+ "Error creating IM entity manager factory with persistence unit: {}",
+ PERSISTENCE_UNIT);
+ throw new IntegrityMonitorException(
+ "Unable to create IM Entity Manager Factory");
}
-
+
// add entry to forward progress and resource registration tables in DB
-
+
// Start a transaction
em = emf.createEntityManager();
- EntityTransaction et = em.getTransaction();
-
- et.begin();
-
- try {
- // if ForwardProgress entry exists for resourceName, update it. If not found, create a new entry
- Query fquery = em.createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
- fquery.setParameter("rn", resourceName);
-
- @SuppressWarnings("rawtypes")
- List fpList = fquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
- ForwardProgressEntity fpx = null;
- if(!fpList.isEmpty()){
- //ignores multiple results
- fpx = (ForwardProgressEntity) fpList.get(0);
- // refresh the object from DB in case cached data was returned
- em.refresh(fpx);
- if(logger.isDebugEnabled()){
- logger.debug("Resource {} exists and will be updated - old fpc= {}, lastUpdated= {}", resourceName, fpx.getFpcCount(), fpx.getLastUpdated());
- }
- fpx.setFpcCount(fpCounter);
- }else{
- //Create a forward progress object
- if(logger.isDebugEnabled()){
- logger.debug("Adding resource {} to ForwardProgress table", resourceName);
- }
- fpx = new ForwardProgressEntity();
- }
- //update/set columns in entry
- fpx.setResourceName(resourceName);
- em.persist(fpx);
- // flush to the DB
- synchronized(IMFLUSHLOCK){
- em.flush();
- }
-
- // if ResourceRegistration entry exists for resourceName, update it. If not found, create a new entry
- Query rquery = em.createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
- rquery.setParameter("rn", resourceName);
-
- @SuppressWarnings("rawtypes")
- List rrList = rquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
- ResourceRegistrationEntity rrx = null;
- if(!rrList.isEmpty()){
- //ignores multiple results
- rrx = (ResourceRegistrationEntity) rrList.get(0);
- // refresh the object from DB in case cached data was returned
- em.refresh(rrx);
- if(logger.isDebugEnabled()){
- logger.debug("Resource {} exists and will be updated - old url= {}, createdDate={}", resourceName, rrx.getResourceUrl(), rrx.getCreatedDate());
- }
- rrx.setLastUpdated(new Date());
- }else{
- // register resource by adding entry to table in DB
- if(logger.isDebugEnabled()){
- logger.debug("Adding resource {} to ResourceRegistration table", resourceName);
- }
- rrx = new ResourceRegistrationEntity();
- }
- //update/set columns in entry
- rrx.setResourceName(resourceName);
- rrx.setResourceUrl(jmxUrl);
- rrx.setNodeType(node_type);
- rrx.setSite(site_name);
- em.persist(rrx);
- // flush to the DB
- synchronized(IMFLUSHLOCK){
- et.commit();
- }
-
- } catch (Exception e) {
- logger.error("IntegrityMonitor constructor DB table update failed with exception: ", e);
- try {
- if (et.isActive()) {
- synchronized(IMFLUSHLOCK){
- et.rollback();
- }
- }
- } catch (Exception e1) {
- logger.error("IntegrityMonitor constructor threw exception: ", e1);
- }
- throw e;
- }
-
+ EntityTransaction et = em.getTransaction();
+
+ et.begin();
+
+ try {
+ // if ForwardProgress entry exists for resourceName, update it. If
+ // not found, create a new entry
+ Query fquery = em
+ .createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
+ fquery.setParameter("rn", resourceName);
+
+ @SuppressWarnings("rawtypes")
+ List fpList = fquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
+ ForwardProgressEntity fpx = null;
+ if (!fpList.isEmpty()) {
+ // ignores multiple results
+ fpx = (ForwardProgressEntity) fpList.get(0);
+ // refresh the object from DB in case cached data was returned
+ em.refresh(fpx);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Resource {} exists and will be updated - old fpc= {}, lastUpdated= {}",
+ resourceName, fpx.getFpcCount(),
+ fpx.getLastUpdated());
+ }
+ fpx.setFpcCount(fpCounter);
+ } else {
+ // Create a forward progress object
+ if (logger.isDebugEnabled()) {
+ logger.debug("Adding resource {} to ForwardProgress table",
+ resourceName);
+ }
+ fpx = new ForwardProgressEntity();
+ }
+ // update/set columns in entry
+ fpx.setResourceName(resourceName);
+ em.persist(fpx);
+ // flush to the DB
+ synchronized (IMFLUSHLOCK) {
+ em.flush();
+ }
+
+ // if ResourceRegistration entry exists for resourceName, update it.
+ // If not found, create a new entry
+ Query rquery = em
+ .createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
+ rquery.setParameter("rn", resourceName);
+
+ @SuppressWarnings("rawtypes")
+ List rrList = rquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
+ ResourceRegistrationEntity rrx = null;
+ if (!rrList.isEmpty()) {
+ // ignores multiple results
+ rrx = (ResourceRegistrationEntity) rrList.get(0);
+ // refresh the object from DB in case cached data was returned
+ em.refresh(rrx);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Resource {} exists and will be updated - old url= {}, createdDate={}",
+ resourceName, rrx.getResourceUrl(),
+ rrx.getCreatedDate());
+ }
+ rrx.setLastUpdated(new Date());
+ } else {
+ // register resource by adding entry to table in DB
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Adding resource {} to ResourceRegistration table",
+ resourceName);
+ }
+ rrx = new ResourceRegistrationEntity();
+ }
+ // update/set columns in entry
+ rrx.setResourceName(resourceName);
+ rrx.setResourceUrl(jmxUrl);
+ rrx.setNodeType(node_type);
+ rrx.setSite(site_name);
+ em.persist(rrx);
+ // flush to the DB
+ synchronized (IMFLUSHLOCK) {
+ et.commit();
+ }
+
+ } catch (Exception e) {
+ logger.error(
+ "IntegrityMonitor constructor DB table update failed with exception: ",
+ e);
+ try {
+ if (et.isActive()) {
+ synchronized (IMFLUSHLOCK) {
+ et.rollback();
+ }
+ }
+ } catch (Exception e1) {
+ logger.error("IntegrityMonitor constructor threw exception: ",
+ e1);
+ }
+ throw e;
+ }
+
// create instance of StateMangement class and pass emf to it
- stateManager = new StateManagement(emf, resourceName);
-
- /**
- * Initialize the state and status attributes. This will maintain any Administrative state value
- * but will set the operational state = enabled, availability status = null, standby status = null.
- * The integrity monitor will set the operational state via the FPManager and the owning application
- * must set the standby status by calling promote/demote on the StateManager.
- */
- stateManager.initializeState();
+ stateManager = new StateManagement(emf, resourceName);
+
+ /**
+ * Initialize the state and status attributes. This will maintain any
+ * Administrative state value but will set the operational state =
+ * enabled, availability status = null, standby status = null. The
+ * integrity monitor will set the operational state via the FPManager
+ * and the owning application must set the standby status by calling
+ * promote/demote on the StateManager.
+ */
+ stateManager.initializeState();
-
// create management bean
try {
new ComponentAdmin(resourceName, this, stateManager);
} catch (Exception e) {
- logger.error("ComponentAdmin constructor exception: {}", e.toString(), e);
+ logger.error("ComponentAdmin constructor exception: {}",
+ e.toString(), e);
}
-
- new FPManager();
-
+
+ fpManager = new FPManager();
}
-
+
private static String getJmxUrl() throws IntegrityMonitorException {
-
+
// get the jmx remote port and construct the JMX URL
Properties systemProps = System.getProperties();
- String jmx_port = systemProps.getProperty("com.sun.management.jmxremote.port");
+ String jmx_port = systemProps
+ .getProperty("com.sun.management.jmxremote.port");
String jmx_err_msg;
if (jmx_port == null) {
jmx_err_msg = "System property com.sun.management.jmxremote.port for JMX remote port is not set";
logger.error("{}", jmx_err_msg);
- throw new IntegrityMonitorException("getJmxUrl exception: " + jmx_err_msg);
+ throw new IntegrityMonitorException("getJmxUrl exception: "
+ + jmx_err_msg);
}
-
+
int port = 0;
try {
port = Integer.parseInt(jmx_port);
} catch (NumberFormatException e) {
- jmx_err_msg = "JMX remote port is not a valid integer value - " + jmx_port;
+ jmx_err_msg = "JMX remote port is not a valid integer value - "
+ + jmx_port;
logger.error("{}", jmx_err_msg);
- throw new IntegrityMonitorException("getJmxUrl exception: " + jmx_err_msg);
+ throw new IntegrityMonitorException("getJmxUrl exception: "
+ + jmx_err_msg);
}
try {
if (jmxFqdn == null) {
- jmxFqdn = InetAddress.getLocalHost().getCanonicalHostName(); // get FQDN of this host
+ jmxFqdn = InetAddress.getLocalHost().getCanonicalHostName(); // get
+ // FQDN
+ // of
+ // this
+ // host
}
} catch (Exception e) {
String msg = "getJmxUrl could not get hostname";
@@ -411,54 +493,68 @@ public class IntegrityMonitor {
}
// assemble the jmx url
- String jmx_url = "service:jmx:rmi:///jndi/rmi://" + jmxFqdn + ":" + port + "/jmxrmi";
-
- if(logger.isDebugEnabled()){
+ String jmx_url = "service:jmx:rmi:///jndi/rmi://" + jmxFqdn + ":"
+ + port + "/jmxrmi";
+
+ if (logger.isDebugEnabled()) {
logger.debug("IntegerityMonitor - jmx url={}", jmx_url);
}
-
+
return jmx_url;
}
+
/**
- * evaluateSanity() is designed to be called by an external entity to evealuate the sanity
- * of the node. It checks the operational and administrative states and the standby
- * status. If the operational state is disabled, it will include the dependencyCheckErrorMsg
- * which includes information about any dependency (node) which has failed.
+ * evaluateSanity() is designed to be called by an external entity to
+ * evealuate the sanity of the node. It checks the operational and
+ * administrative states and the standby status. If the operational state is
+ * disabled, it will include the dependencyCheckErrorMsg which includes
+ * information about any dependency (node) which has failed.
*/
public void evaluateSanity() throws Exception {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("evaluateSanity called ....");
}
- synchronized(evaluateSanityLock){
+ synchronized (evaluateSanityLock) {
String error_msg = dependencyCheckErrorMsg;
- if(logger.isDebugEnabled()){
- logger.debug("evaluateSanity dependencyCheckErrorMsg = {}", error_msg);
+ if (logger.isDebugEnabled()) {
+ logger.debug("evaluateSanity dependencyCheckErrorMsg = {}",
+ error_msg);
}
// check op state and throw exception if disabled
- if ((stateManager.getOpState() != null) && stateManager.getOpState().equals(StateManagement.DISABLED)) {
- String msg = "Resource " + resourceName + " operation state is disabled. " + error_msg;
- if(logger.isDebugEnabled()){
+ if ((stateManager.getOpState() != null)
+ && stateManager.getOpState().equals(
+ StateManagement.DISABLED)) {
+ String msg = "Resource " + resourceName
+ + " operation state is disabled. " + error_msg;
+ if (logger.isDebugEnabled()) {
logger.debug("{}", msg);
}
throw new IntegrityMonitorException(msg);
}
// check admin state and throw exception if locked
- if ((stateManager.getAdminState() != null) && stateManager.getAdminState().equals(StateManagement.LOCKED)) {
- String msg = "Resource " + resourceName + " is administratively locked";
- if(logger.isDebugEnabled()){
+ if ((stateManager.getAdminState() != null)
+ && stateManager.getAdminState().equals(
+ StateManagement.LOCKED)) {
+ String msg = "Resource " + resourceName
+ + " is administratively locked";
+ if (logger.isDebugEnabled()) {
logger.debug("{}", msg);
}
- throw new AdministrativeStateException("IntegrityMonitor Admin State Exception: " + msg);
+ throw new AdministrativeStateException(
+ "IntegrityMonitor Admin State Exception: " + msg);
}
// check standby state and throw exception if cold standby
- if ((stateManager.getStandbyStatus() != null) && stateManager.getStandbyStatus().equals(StateManagement.COLD_STANDBY)){
+ if ((stateManager.getStandbyStatus() != null)
+ && stateManager.getStandbyStatus().equals(
+ StateManagement.COLD_STANDBY)) {
String msg = "Resource " + resourceName + " is cold standby";
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("{}", msg);
}
- throw new StandbyStatusException("IntegrityMonitor Standby Status Exception: " + msg);
+ throw new StandbyStatusException(
+ "IntegrityMonitor Standby Status Exception: " + msg);
}
}
@@ -466,244 +562,283 @@ public class IntegrityMonitor {
}
/*
- * This method checks the forward progress counter and the state of
- * a dependency. If the dependency is unavailable or failed, an
- * error message is created which is checked when evaluateSanity interface
- * is called. If the error message is set then the evaluateSanity
- * will return an error.
+ * This method checks the forward progress counter and the state of a
+ * dependency. If the dependency is unavailable or failed, an error message
+ * is created which is checked when evaluateSanity interface is called. If
+ * the error message is set then the evaluateSanity will return an error.
*/
public String stateCheck(String dep) {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("checking state of dependent resource: {}", dep);
}
String error_msg = null;
ForwardProgressEntity forwardProgressEntity = null;
StateManagementEntity stateManagementEntity = null;
-
+
// Start a transaction
EntityTransaction et = em.getTransaction();
et.begin();
- try{
- Query query = em.createQuery("Select p from ForwardProgressEntity p where p.resourceName=:resource");
+ try {
+ Query query = em
+ .createQuery("Select p from ForwardProgressEntity p where p.resourceName=:resource");
query.setParameter("resource", dep);
@SuppressWarnings("rawtypes")
- List fpList = query.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ List fpList = query.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
if (!fpList.isEmpty()) {
// exists
forwardProgressEntity = (ForwardProgressEntity) fpList.get(0);
// refresh the object from DB in case cached data was returned
em.refresh(forwardProgressEntity);
- if(logger.isDebugEnabled()){
- logger.debug("Found entry in ForwardProgressEntity table for dependent Resource={}", dep);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Found entry in ForwardProgressEntity table for dependent Resource={}",
+ dep);
}
} else {
- error_msg = dep + ": resource not found in ForwardProgressEntity database table";
+ error_msg = dep
+ + ": resource not found in ForwardProgressEntity database table";
logger.error("{}", error_msg);
}
- synchronized(IMFLUSHLOCK){
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
- }
- catch(Exception ex){
+ } catch (Exception ex) {
// log an error
- error_msg = dep + ": ForwardProgressEntity DB operation failed with exception: ";
+ error_msg = dep
+ + ": ForwardProgressEntity DB operation failed with exception: ";
logger.error("{}", error_msg, ex);
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.rollback();
}
}
}
- if(error_msg==null){
+ if (error_msg == null) {
// Start a transaction
et = em.getTransaction();
et.begin();
try {
// query if StateManagement entry exists for dependent resource
- Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+ Query query = em
+ .createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
query.setParameter("resource", dep);
@SuppressWarnings("rawtypes")
- List smList = query.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ List smList = query.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
if (!smList.isEmpty()) {
- // exist
- stateManagementEntity = (StateManagementEntity) smList.get(0);
- // refresh the object from DB in case cached data was returned
+ // exist
+ stateManagementEntity = (StateManagementEntity) smList
+ .get(0);
+ // refresh the object from DB in case cached data was
+ // returned
em.refresh(stateManagementEntity);
- if(logger.isDebugEnabled()){
- logger.debug("Found entry in StateManagementEntity table for dependent Resource={}", dep);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Found entry in StateManagementEntity table for dependent Resource={}",
+ dep);
}
} else {
- error_msg = dep + ": resource not found in state management entity database table";
+ error_msg = dep
+ + ": resource not found in state management entity database table";
logger.error("{}", error_msg);
}
- synchronized(IMFLUSHLOCK){
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
} catch (Exception e) {
// log an error
- error_msg = dep + ": StateManagementEntity DB read failed with exception: ";
+ error_msg = dep
+ + ": StateManagementEntity DB read failed with exception: ";
logger.error("{}", error_msg, e);
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.rollback();
}
}
}
}
- //verify that the ForwardProgress is current (check last_updated)
- if(error_msg==null){
+ // verify that the ForwardProgress is current (check last_updated)
+ if (error_msg == null) {
if (forwardProgressEntity != null && stateManagementEntity != null) {
Date date = new Date();
- long diffMs = date.getTime() - forwardProgressEntity.getLastUpdated().getTime();
- if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateCheck(): diffMs = {}", diffMs);
+ long diffMs = date.getTime()
+ - forwardProgressEntity.getLastUpdated().getTime();
+ if (logger.isDebugEnabled()) {
+ logger.debug("IntegrityMonitor.stateCheck(): diffMs = {}",
+ diffMs);
}
- //Threshold for a stale entry
- long staleMs = failedCounterThreshold * monitorInterval * (long)1000;
- if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateCheck(): staleMs = {}", staleMs);
+ // Threshold for a stale entry
+ long staleMs = 1000L * maxFpcUpdateInterval;
+ if (logger.isDebugEnabled()) {
+ logger.debug("IntegrityMonitor.stateCheck(): staleMs = {}",
+ staleMs);
}
- if(diffMs > staleMs){
- //ForwardProgress is stale. Disable it
+ if (diffMs > staleMs) {
+ // ForwardProgress is stale. Disable it
try {
- if(!stateManagementEntity.getOpState().equals(StateManagement.DISABLED)){
- if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateCheck(): Changing OpStat = disabled for {}", dep);
+ if (!stateManagementEntity.getOpState().equals(
+ StateManagement.DISABLED)) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "IntegrityMonitor.stateCheck(): Changing OpStat = disabled for {}",
+ dep);
}
stateManager.disableFailed(dep);
}
} catch (Exception e) {
- String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = " + dep
- + "; " + e.getMessage();
- logger.error("{}",msg, e);
+ String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = "
+ + dep + "; " + e.getMessage();
+ logger.error("{}", msg, e);
}
}
- }
- else {
-
- if(forwardProgressEntity == null) {
- String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = " + dep
- + "; " + " forwardProgressEntity == null.";
+ } else {
+
+ if (forwardProgressEntity == null) {
+ String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = "
+ + dep + "; " + " forwardProgressEntity == null.";
logger.error("{}", msg);
}
-
- else {
- String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = " + dep
- + "; " + " stateManagementEntity == null.";
+
+ else {
+ String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = "
+ + dep + "; " + " stateManagementEntity == null.";
logger.error("{}", msg);
}
}
}
-
+
// check operation, admin and standby states of dependent resource
if (error_msg == null) {
- if(stateManagementEntity != null) {
- if ((stateManager.getAdminState() != null) && stateManagementEntity.getAdminState().equals(StateManagement.LOCKED)) {
+ if (stateManagementEntity != null) {
+ if ((stateManager.getAdminState() != null)
+ && stateManagementEntity.getAdminState().equals(
+ StateManagement.LOCKED)) {
error_msg = dep + ": resource is administratively locked";
logger.error("{}", error_msg);
- } else if ((stateManager.getOpState() != null) && stateManagementEntity.getOpState().equals(StateManagement.DISABLED)) {
+ } else if ((stateManager.getOpState() != null)
+ && stateManagementEntity.getOpState().equals(
+ StateManagement.DISABLED)) {
error_msg = dep + ": resource is operationally disabled";
logger.error("{}", error_msg);
- } else if ((stateManager.getStandbyStatus() != null) && stateManagementEntity.getStandbyStatus().equals(StateManagement.COLD_STANDBY)) {
+ } else if ((stateManager.getStandbyStatus() != null)
+ && stateManagementEntity.getStandbyStatus().equals(
+ StateManagement.COLD_STANDBY)) {
error_msg = dep + ": resource is cold standby";
logger.error("{}", error_msg);
}
- }
- else {
- error_msg = dep + ": could not check standy state of resource. stateManagementEntity == null.";
+ } else {
+ error_msg = dep
+ + ": could not check standy state of resource. stateManagementEntity == null.";
logger.error("{}", error_msg);
}
}
-
- String returnMsg = "IntegrityMonitor.stateCheck(): returned error_msg: " + error_msg;
- if(logger.isDebugEnabled()){
+
+ String returnMsg = "IntegrityMonitor.stateCheck(): returned error_msg: "
+ + error_msg;
+ if (logger.isDebugEnabled()) {
logger.debug("{}", returnMsg);
}
return error_msg;
}
-
+
private String fpCheck(String dep) {
- if(logger.isDebugEnabled()){
- logger.debug("checking forward progress count of dependent resource: {}", dep);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "checking forward progress count of dependent resource: {}",
+ dep);
}
-
+
String error_msg = null;
-
- // check FPC count - a changing FPC count indicates the resource JVM is running
+
+ // check FPC count - a changing FPC count indicates the resource JVM is
+ // running
// Start a transaction
EntityTransaction et = em.getTransaction();
et.begin();
try {
- Query fquery = em.createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
+ Query fquery = em
+ .createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
fquery.setParameter("rn", dep);
@SuppressWarnings("rawtypes")
- List fpList = fquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ List fpList = fquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
ForwardProgressEntity fpx;
if (!fpList.isEmpty()) {
- //ignores multiple results
+ // ignores multiple results
fpx = (ForwardProgressEntity) fpList.get(0);
// refresh the object from DB in case cached data was returned
em.refresh(fpx);
- if(logger.isDebugEnabled()){
- logger.debug("Dependent resource {} - fpc= {}, lastUpdated={}", dep, fpx.getFpcCount(), fpx.getLastUpdated());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Dependent resource {} - fpc= {}, lastUpdated={}",
+ dep, fpx.getFpcCount(), fpx.getLastUpdated());
}
long currTime = System.currentTimeMillis();
- // if dependent resource FPC has not been updated, consider it an error
+ // if dependent resource FPC has not been updated, consider it
+ // an error
if ((currTime - fpx.getLastUpdated().getTime()) > (1000 * maxFpcUpdateInterval)) {
- error_msg = dep + ": FP count has not been updated in the last " + maxFpcUpdateInterval + " seconds";
+ error_msg = dep
+ + ": FP count has not been updated in the last "
+ + maxFpcUpdateInterval + " seconds";
logger.error("{}", error_msg);
try {
// create instance of StateMangement class for dependent
- StateManagement depStateManager = new StateManagement(emf, dep);
- if(!depStateManager.getOpState().equals(StateManagement.DISABLED)){
- if(logger.isDebugEnabled()){
- logger.debug("Forward progress not detected for dependent resource {}. Setting dependent's state to disable failed.", dep);
- }
- depStateManager.disableFailed();
+ StateManagement depStateManager = new StateManagement(
+ emf, dep);
+ if (!depStateManager.getOpState().equals(
+ StateManagement.DISABLED)) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Forward progress not detected for dependent resource {}. Setting dependent's state to disable failed.",
+ dep);
}
+ depStateManager.disableFailed();
+ }
} catch (Exception e) {
// ignore errors
- logger.error("Update dependent state failed with exception: ", e);
+ logger.error(
+ "Update dependent state failed with exception: ",
+ e);
}
}
} else {
// resource entry not found in FPC table
- error_msg = dep + ": resource not found in ForwardProgressEntity table in the DB";
+ error_msg = dep
+ + ": resource not found in ForwardProgressEntity table in the DB";
logger.error("{}", error_msg);
}
- synchronized(IMFLUSHLOCK){
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
} catch (Exception e) {
// log an error and continue
- error_msg = dep + ": ForwardProgressEntity DB read failed with exception: ";
+ error_msg = dep
+ + ": ForwardProgressEntity DB read failed with exception: ";
logger.error("{}", error_msg, e);
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.rollback();
}
}
}
-
+
return error_msg;
}
-
- public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity(){
- if(logger.isDebugEnabled()){
+
+ public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity() {
+ if (logger.isDebugEnabled()) {
logger.debug("getAllForwardProgressEntity: entry");
}
ArrayList<ForwardProgressEntity> fpList = new ArrayList<>();
@@ -711,27 +846,31 @@ public class IntegrityMonitor {
EntityTransaction et = em.getTransaction();
et.begin();
try {
- Query fquery = em.createQuery("Select e from ForwardProgressEntity e");
+ Query fquery = em
+ .createQuery("Select e from ForwardProgressEntity e");
@SuppressWarnings("rawtypes")
- List myList = fquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
- synchronized(IMFLUSHLOCK){
+ List myList = fquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
- if(logger.isDebugEnabled()){
- logger.debug("getAllForwardProgressEntity: myList.size(): {}", myList.size());
- }
- if(!myList.isEmpty()){
- for(int i = 0; i < myList.size(); i++){
- if(logger.isDebugEnabled()){
- logger.debug("getAllForwardProgressEntity: myList.get({}).getResourceName(): {}",i,
- ((ForwardProgressEntity)myList.get(i)).getResourceName());
+ if (logger.isDebugEnabled()) {
+ logger.debug("getAllForwardProgressEntity: myList.size(): {}",
+ myList.size());
+ }
+ if (!myList.isEmpty()) {
+ for (int i = 0; i < myList.size(); i++) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "getAllForwardProgressEntity: myList.get({}).getResourceName(): {}",
+ i, ((ForwardProgressEntity) myList.get(i))
+ .getResourceName());
}
fpList.add((ForwardProgressEntity) myList.get(i));
}
}
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.commit();
}
}
@@ -739,19 +878,20 @@ public class IntegrityMonitor {
// log an error and continue
String msg = "getAllForwardProgessEntity DB read failed with exception: ";
logger.error("{}", msg, e);
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.rollback();
}
}
}
return fpList;
}
-
-
+
private String jmxCheck(String dep) {
- if(logger.isDebugEnabled()){
- logger.debug("checking health of dependent by calling test() via JMX on resource: {}", dep);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "checking health of dependent by calling test() via JMX on resource: {}",
+ dep);
}
String error_msg = null;
@@ -763,57 +903,65 @@ public class IntegrityMonitor {
et.begin();
try {
// query if ResourceRegistration entry exists for resourceName
- Query rquery = em.createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
+ Query rquery = em
+ .createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
rquery.setParameter("rn", dep);
@SuppressWarnings("rawtypes")
- List rrList = rquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ List rrList = rquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
ResourceRegistrationEntity rrx = null;
if (!rrList.isEmpty()) {
- //ignores multiple results
+ // ignores multiple results
rrx = (ResourceRegistrationEntity) rrList.get(0);
// refresh the object from DB in case cached data was returned
- em.refresh(rrx);
+ em.refresh(rrx);
jmxUrl = rrx.getResourceUrl();
- if(logger.isDebugEnabled()){
- logger.debug("Dependent Resource={}, url={}, createdDate={}", dep, jmxUrl, rrx.getCreatedDate());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Dependent Resource={}, url={}, createdDate={}",
+ dep, jmxUrl, rrx.getCreatedDate());
}
} else {
- error_msg = dep + ": resource not found in ResourceRegistrationEntity table in the DB";
+ error_msg = dep
+ + ": resource not found in ResourceRegistrationEntity table in the DB";
logger.error("{}", error_msg);
}
- synchronized(IMFLUSHLOCK){
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
} catch (Exception e) {
- error_msg = dep + ": ResourceRegistrationEntity DB read failed with exception: ";
+ error_msg = dep
+ + ": ResourceRegistrationEntity DB read failed with exception: ";
logger.error("{}", error_msg, e);
- synchronized(IMFLUSHLOCK){
- if(et.isActive()){
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
et.rollback();
}
}
}
-
if (jmxUrl != null) {
JmxAgentConnection jmxAgentConnection = null;
try {
jmxAgentConnection = new JmxAgentConnection(jmxUrl);
- MBeanServerConnection mbeanServer = jmxAgentConnection.getMBeanConnection();
- ComponentAdminMBean admin = JMX.newMXBeanProxy(mbeanServer, ComponentAdmin.getObjectName(dep),
+ MBeanServerConnection mbeanServer = jmxAgentConnection
+ .getMBeanConnection();
+ ComponentAdminMBean admin = JMX.newMXBeanProxy(mbeanServer,
+ ComponentAdmin.getObjectName(dep),
ComponentAdminMBean.class);
// invoke the test method via the jmx proxy
admin.test();
- if(logger.isDebugEnabled()){
- logger.debug("Dependent resource {} sanity test passed", dep);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Dependent resource {} sanity test passed",
+ dep);
}
} catch (Exception e) {
- error_msg = dep + ": resource sanity test failed with exception: ";
+ error_msg = dep
+ + ": resource sanity test failed with exception: ";
logger.error("{}", error_msg, e);
} finally {
// close the JMX connector
@@ -825,46 +973,53 @@ public class IntegrityMonitor {
return error_msg;
}
-
- private String dependencyCheck() {
- if(logger.isDebugEnabled()){
- logger.debug("dependencyCheck: entry - checking health of dependent groups and setting resource's state");
+
+ public String dependencyCheck() {
+ if (logger.isDebugEnabled()) {
+ logger.debug("dependencyCheck: entry");
}
- synchronized(dependencyCheckLock){
+ synchronized (dependencyCheckLock) {
// Start with the error message empty
String error_msg = "";
boolean dependencyFailure = false;
-
+
/*
* Before we check dependency groups we need to check subsystemTest.
*/
try {
- //Test any subsystems that are not covered under the dependency relationship
+ // Test any subsystems that are not covered under the dependency
+ // relationship
subsystemTest();
- }catch (Exception e){
+ } catch (Exception e) {
logger.error("IntegrityMonitor threw exception", e);
- dependencyFailure=true;
- //This indicates a subsystemTest failure
+ dependencyFailure = true;
+ // This indicates a subsystemTest failure
try {
- if(logger.isDebugEnabled()){
- logger.debug("{}: There has been a subsystemTest failure with error:{} Updating this resource's state to disableDependency", resourceName, e.getMessage());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "{}: There has been a subsystemTest failure with error:{} Updating this resource's state to disableDependency",
+ resourceName, e.getMessage());
}
- //Capture the subsystemTest failure info
- if(!error_msg.isEmpty()){
+ // Capture the subsystemTest failure info
+ if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(",");
}
- error_msg = error_msg.concat(resourceName + ": " + e.getMessage());
+ error_msg = error_msg.concat(resourceName + ": "
+ + e.getMessage());
this.stateManager.disableDependency();
} catch (Exception ex) {
logger.error("IntegrityMonitor threw exception.", ex);
if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(",");
}
- error_msg = error_msg.concat("\n" + resourceName + ": Failed to disable dependency after subsystemTest failure due to: " + ex.getMessage());
- }
+ error_msg = error_msg
+ .concat("\n"
+ + resourceName
+ + ": Failed to disable dependency after subsystemTest failure due to: "
+ + ex.getMessage());
+ }
}
-
// Check the sanity of dependents for lead subcomponents
if (dep_groups != null && dep_groups.length > 0) {
@@ -875,9 +1030,10 @@ public class IntegrityMonitor {
// ignore empty group
continue;
}
- String [] dependencies = group.split(",");
- if(logger.isDebugEnabled()){
- logger.debug("group dependencies = {}", Arrays.toString(dependencies));
+ String[] dependencies = group.split(",");
+ if (logger.isDebugEnabled()) {
+ logger.debug("group dependencies = {}",
+ Arrays.toString(dependencies));
}
int real_dep_count = 0;
int fail_dep_count = 0;
@@ -887,8 +1043,12 @@ public class IntegrityMonitor {
// ignore empty dependency
continue;
}
- real_dep_count++; // this is a valid dependency whose state is tracked
- String fail_msg = fpCheck(dep); // if a resource is down, its FP count will not be incremented
+ real_dep_count++; // this is a valid dependency whose
+ // state is tracked
+ String fail_msg = fpCheck(dep); // if a resource is
+ // down, its FP count
+ // will not be
+ // incremented
if (fail_msg == null) {
if (testViaJmx) {
fail_msg = jmxCheck(dep);
@@ -901,51 +1061,50 @@ public class IntegrityMonitor {
if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(", ");
}
- error_msg = error_msg.concat(fail_msg);
+ error_msg = error_msg.concat(fail_msg);
}
- }// end for (String dep : dependencies)
+ }// end for (String dep : dependencies)
- // if all dependencies in a group are failed, set this resource's state to disable dependency
- if ((real_dep_count > 0) && (fail_dep_count == real_dep_count)) {
- dependencyFailure=true;
+ // if all dependencies in a group are failed, set this
+ // resource's state to disable dependency
+ if ((real_dep_count > 0)
+ && (fail_dep_count == real_dep_count)) {
+ dependencyFailure = true;
try {
if(logger.isDebugEnabled()){
logger.debug("All dependents in group {} have failed their health check. Updating this resource's state to disableDependency", group);
- }
- if( !( (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
+ }
+ if(stateManager.getAvailStatus()== null || !( (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
(stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY_FAILED) ) ){
// Note: redundant calls are made by refreshStateAudit
this.stateManager.disableDependency();
- }else //corruption has occurred - This will not be corrected by the refreshStateAudit
- if(!(stateManager.getOpState()).equals(StateManagement.DISABLED)){
- // Note: redundant calls are made by refreshStateAudit
- this.stateManager.disableDependency();
- }
+ }
} catch (Exception e) {
logger.error("IntegrityMonitor threw exception.", e);
if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(",");
}
- error_msg = error_msg.concat(resourceName + ": Failed to disable dependency");
- break; // break out on failure and skip checking other groups
+ error_msg = error_msg.concat(resourceName
+ + ": Failed to disable dependency");
+ break; // break out on failure and skip checking
+ // other groups
}
}
- //check the next group
+ // check the next group
- }//end for (String group : dep_groups)
-
+ }// end for (String group : dep_groups)
/*
- * We have checked all the dependency groups. If all are ok and subsystemTest passed,
- * dependencyFailure == false
+ * We have checked all the dependency groups. If all are ok and
+ * subsystemTest passed, dependencyFailure == false
*/
- if(!dependencyFailure){
+ if (!dependencyFailure) {
try {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("All dependency groups have at least one viable member. Updating this resource's state to enableNoDependency");
}
- if( (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
- (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY_FAILED) ){
+ if(stateManager.getAvailStatus() != null && ((stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
+ (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY_FAILED)) ){
// Note: redundant calls are made by refreshStateAudit
this.stateManager.enableNoDependency();
} // The refreshStateAudit will catch the case where it is disabled but availStatus != failed
@@ -954,320 +1113,425 @@ public class IntegrityMonitor {
if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(",");
}
- error_msg = error_msg.concat(resourceName + ": Failed to enable no dependency");
+ error_msg = error_msg.concat(resourceName
+ + ": Failed to enable no dependency");
}
}
- }else if(!dependencyFailure){
+ } else if (!dependencyFailure) {
/*
- * This is put here to clean up when no dependency group should exist, but one was erroneously
- * added which caused the state to be disabled/dependency/coldstandby and later removed. We saw
- * this happen in the lab, but is not very likely in a production environment...but you never know.
+ * This is put here to clean up when no dependency group should
+ * exist, but one was erroneously added which caused the state
+ * to be disabled/dependency/coldstandby and later removed. We
+ * saw this happen in the lab, but is not very likely in a
+ * production environment...but you never know.
*/
try {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("There are no dependents. Updating this resource's state to enableNoDependency");
}
- if( (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
- (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY_FAILED) ){
+ if(stateManager.getAvailStatus() != null && ((stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY) ||
+ (stateManager.getAvailStatus()).equals(StateManagement.DEPENDENCY_FAILED)) ){
// Note: redundant calls are made by refreshStateAudit
- this.stateManager.enableNoDependency();
- }// The refreshStateAudit will catch the case where it is disabled but availStatus != failed
+ this.stateManager.enableNoDependency();
+ }// The refreshStateAudit will catch the case where it is
+ // disabled but availStatus != failed
} catch (Exception e) {
logger.error("IntegrityMonitor threw exception.", e);
if (!error_msg.isEmpty()) {
error_msg = error_msg.concat(",");
}
- error_msg = error_msg.concat(resourceName + ": Failed to enable no dependency");
+ error_msg = error_msg.concat(resourceName
+ + ": Failed to enable no dependency");
}
}
-
-
if (!error_msg.isEmpty()) {
- logger.error("Sanity failure detected in a dependent resource: {}", error_msg);
+ logger.error(
+ "Sanity failure detected in a dependent resource: {}",
+ error_msg);
}
-
+
dependencyCheckErrorMsg = error_msg;
lastDependencyCheckTime = System.currentTimeMillis();
+ if (logger.isDebugEnabled()) {
+ logger.debug("dependencyCheck: exit");
+ }
return error_msg;
}
}
-
+
/**
* Execute a test transaction. It is called when the test transaction timer fires.
* It could be overridden to provide additional test functionality. If overridden,
- * the overriding method must invoke startTransaction() and endTransaction()
+ * the overriding method must invoke startTransaction() and endTransaction() and
+ * check if the allNotWellMap is empty.
*/
public void testTransaction() {
synchronized (testTransactionLock){
if(logger.isDebugEnabled()){
- logger.debug("testTransaction called...");
+ logger.debug("testTransaction: entry");
}
-
//
// startTransaction() not required for testTransaction
- //
+ //
// end transaction - increments local FP counter
endTransaction();
}
}
-
+
/**
- * Additional testing for subsystems that do not have a /test interface (for ex. 3rd party
- * processes like elk). This method would be overridden by the subsystem.
+ * Additional testing for subsystems that do not have a /test interface (for
+ * ex. 3rd party processes like elk). This method would be overridden by the
+ * subsystem.
*/
public void subsystemTest() throws IntegrityMonitorException {
// Testing provided by subsystem
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("IntegrityMonitor subsystemTest() OK");
}
}
-
+
/**
- * Checks admin state and resets transaction timer.
- * Called by application at the start of a transaction.
- * @throws AdministrativeStateException throws admin state exception if resource is locked
- * @throws StandbyStatusException
+ * Checks admin state and resets transaction timer. Called by application at
+ * the start of a transaction.
+ *
+ * @throws AdministrativeStateException
+ * throws admin state exception if resource is locked
+ * @throws StandbyStatusException
*/
- public void startTransaction() throws AdministrativeStateException, StandbyStatusException {
+ public void startTransaction() throws AdministrativeStateException,
+ StandbyStatusException {
- synchronized(startTransactionLock){
+ synchronized (startTransactionLock) {
// check admin state and throw exception if locked
- if ((stateManager.getAdminState() != null) && stateManager.getAdminState().equals(StateManagement.LOCKED)) {
- String msg = "Resource " + resourceName + " is administratively locked";
+ if ((stateManager.getAdminState() != null)
+ && stateManager.getAdminState().equals(
+ StateManagement.LOCKED)) {
+ String msg = "Resource " + resourceName
+ + " is administratively locked";
- throw new AdministrativeStateException("IntegrityMonitor Admin State Exception: " + msg);
+ throw new AdministrativeStateException(
+ "IntegrityMonitor Admin State Exception: " + msg);
}
// check standby state and throw exception if locked
- if ((stateManager.getStandbyStatus() != null) &&
- (stateManager.getStandbyStatus().equals(StateManagement.HOT_STANDBY) ||
- stateManager.getStandbyStatus().equals(StateManagement.COLD_STANDBY))){
+ if ((stateManager.getStandbyStatus() != null)
+ && (stateManager.getStandbyStatus().equals(
+ StateManagement.HOT_STANDBY) || stateManager
+ .getStandbyStatus().equals(
+ StateManagement.COLD_STANDBY))) {
String msg = "Resource " + resourceName + " is standby";
- throw new StandbyStatusException("IntegrityMonitor Standby Status Exception: " + msg);
+ throw new StandbyStatusException(
+ "IntegrityMonitor Standby Status Exception: " + msg);
}
// reset transactionTimer so it will not fire
elapsedTestTransTime = 0;
}
}
-
+
/**
- * Increment the local forward progress counter. Called by application at the
- * end of each transaction (successful or not).
+ * Increment the local forward progress counter. Called by application at
+ * the end of each transaction (successful or not).
*/
public void endTransaction() {
synchronized(endTransactionLock){
+ if(getAllNotWellMap() != null){
+ if(!(getAllNotWellMap().isEmpty())){
+ /*
+ * An entity has reported that it is not well. We must not
+ * allow the the forward progress counter to advance.
+ */
+ String msg = "allNotWellMap:";
+ for(String key: allNotWellMap.keySet()){
+ msg = msg.concat("\nkey = " + key + " msg = " + allNotWellMap.get(key));
+ }
+ logger.error("endTransaction: allNotWellMap is NOT EMPTY. Not advancing forward"
+ + "progress counter. \n{}\n", msg);
+ return;
+ }else{
+ if(logger.isDebugEnabled()){
+ if(getAllSeemsWellMap() != null){
+ if(!(getAllSeemsWellMap().isEmpty())){
+ String msg = "allSeemsWellMap:";
+ for(String key: allSeemsWellMap.keySet()){
+ msg = msg.concat("\nkey = " + key + " msg = " + allSeemsWellMap.get(key));
+ }
+ logger.debug("endTransaction: allNotWellMap IS EMPTY and allSeemsWellMap is NOT EMPTY. Advancing forward"
+ + "progress counter. \n{}\n", msg);
+ }
+ }
+ }
+ }
+ }
// increment local FPC
fpCounter++;
}
}
-
+
// update FP count in DB with local FP count
private void writeFpc() throws IntegrityMonitorException {
-
+
// Start a transaction
EntityTransaction et = em.getTransaction();
- if(!et.isActive()){
+ if (!et.isActive()) {
et.begin();
}
try {
// query if ForwardProgress entry exists for resourceName
- Query fquery = em.createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
+ Query fquery = em
+ .createQuery("Select f from ForwardProgressEntity f where f.resourceName=:rn");
fquery.setParameter("rn", resourceName);
@SuppressWarnings("rawtypes")
- List fpList = fquery.setLockMode(
- LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ List fpList = fquery.setLockMode(LockModeType.NONE)
+ .setFlushMode(FlushModeType.COMMIT).getResultList();
ForwardProgressEntity fpx;
- if(!fpList.isEmpty()) {
- //ignores multiple results
+ if (!fpList.isEmpty()) {
+ // ignores multiple results
fpx = (ForwardProgressEntity) fpList.get(0);
// refresh the object from DB in case cached data was returned
- em.refresh(fpx);
- if(logger.isDebugEnabled()){
- logger.debug("Updating FP entry: Resource={}, fpcCount={}, lastUpdated={}, new fpcCount={}",
- resourceName,
- fpx.getFpcCount(),
- fpx.getLastUpdated(),
- fpCounter);
- }
+ em.refresh(fpx);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Updating FP entry: Resource={}, fpcCount={}, lastUpdated={}, new fpcCount={}",
+ resourceName, fpx.getFpcCount(),
+ fpx.getLastUpdated(), fpCounter);
+ }
fpx.setFpcCount(fpCounter);
em.persist(fpx);
// flush to the DB and commit
- synchronized(IMFLUSHLOCK){
+ synchronized (IMFLUSHLOCK) {
et.commit();
}
- }
- else {
+ } else {
// Error - FP entry does not exist
- String msg = "FP entry not found in database for resource " + resourceName;
+ String msg = "FP entry not found in database for resource "
+ + resourceName;
throw new IntegrityMonitorException(msg);
}
- } catch (Exception e) {
- try {
- synchronized(IMFLUSHLOCK){
- if (et.isActive()) {
- et.rollback();
- }
- }
- } catch (Exception e1) {
- logger.error("IntegrityMonitor threw exception.", e1);
- }
- logger.error("writeFpc DB table commit failed with exception: {}", e);
- throw e;
- }
+ } catch (Exception e) {
+ try {
+ synchronized (IMFLUSHLOCK) {
+ if (et.isActive()) {
+ et.rollback();
+ }
+ }
+ } catch (Exception e1) {
+ logger.error("IntegrityMonitor threw exception.", e1);
+ }
+ logger.error("writeFpc DB table commit failed with exception: {}",
+ e);
+ throw e;
+ }
}
-
+
// retrieve state manager reference
public final StateManagement getStateManager() {
return this.stateManager;
}
-
+
/**
* Read and validate properties
- * @throws Exception
+ *
+ * @throws Exception
*/
- private static void validateProperties(Properties prop) throws IntegrityMonitorPropertiesException {
-
- if (prop.getProperty(IntegrityMonitorProperties.DB_DRIVER)== null){
- String msg = IntegrityMonitorProperties.DB_DRIVER + " property is null";
+ private static void validateProperties(Properties prop)
+ throws IntegrityMonitorPropertiesException {
+
+ if (prop.getProperty(IntegrityMonitorProperties.DB_DRIVER) == null) {
+ String msg = IntegrityMonitorProperties.DB_DRIVER
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
}
-
- if (prop.getProperty(IntegrityMonitorProperties.DB_URL)== null){
- String msg = IntegrityMonitorProperties.DB_URL + " property is null";
+
+ if (prop.getProperty(IntegrityMonitorProperties.DB_URL) == null) {
+ String msg = IntegrityMonitorProperties.DB_URL
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
}
-
- if (prop.getProperty(IntegrityMonitorProperties.DB_USER)== null){
- String msg = IntegrityMonitorProperties.DB_USER + " property is null";
+
+ if (prop.getProperty(IntegrityMonitorProperties.DB_USER) == null) {
+ String msg = IntegrityMonitorProperties.DB_USER
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
}
-
- if (prop.getProperty(IntegrityMonitorProperties.DB_PWD)== null){
- String msg = IntegrityMonitorProperties.DB_PWD + " property is null";
+
+ if (prop.getProperty(IntegrityMonitorProperties.DB_PWD) == null) {
+ String msg = IntegrityMonitorProperties.DB_PWD
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
}
-
+
if (prop.getProperty(IntegrityMonitorProperties.FP_MONITOR_INTERVAL) != null) {
try {
- monitorInterval = Integer.parseInt(prop.getProperty(IntegrityMonitorProperties.FP_MONITOR_INTERVAL).trim());
+ monitorInterval = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.FP_MONITOR_INTERVAL).trim());
} catch (NumberFormatException e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.FP_MONITOR_INTERVAL, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.FP_MONITOR_INTERVAL, e);
}
}
-
+
if (prop.getProperty(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD) != null) {
try {
- failedCounterThreshold = Integer.parseInt(prop.getProperty(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD).trim());
+ failedCounterThreshold = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD)
+ .trim());
} catch (NumberFormatException e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, e);
}
}
-
+
if (prop.getProperty(IntegrityMonitorProperties.TEST_TRANS_INTERVAL) != null) {
try {
- testTransInterval = Integer.parseInt(prop.getProperty(IntegrityMonitorProperties.TEST_TRANS_INTERVAL).trim());
+ testTransInterval = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.TEST_TRANS_INTERVAL).trim());
} catch (NumberFormatException e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.TEST_TRANS_INTERVAL, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.TEST_TRANS_INTERVAL, e);
}
}
-
+
if (prop.getProperty(IntegrityMonitorProperties.WRITE_FPC_INTERVAL) != null) {
try {
- writeFpcInterval = Integer.parseInt(prop.getProperty(IntegrityMonitorProperties.WRITE_FPC_INTERVAL).trim());
+ writeFpcInterval = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.WRITE_FPC_INTERVAL).trim());
} catch (NumberFormatException e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.WRITE_FPC_INTERVAL, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.WRITE_FPC_INTERVAL, e);
}
}
-
+
+ if (prop.getProperty(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL) != null) {
+ try {
+ checkDependencyInterval = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL).trim());
+ } catch (NumberFormatException e) {
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, e);
+ }
+ }
+
// dependency_groups are a semi-colon separated list of groups
// each group is a comma separated list of resource names
- // For ex. dependency_groups = site_1.pap_1,site_1.pap_2 ; site_1.pdp_1, site_1.pdp_2
+ // For ex. dependency_groups = site_1.pap_1,site_1.pap_2 ; site_1.pdp_1,
+ // site_1.pdp_2
if (prop.getProperty(IntegrityMonitorProperties.DEPENDENCY_GROUPS) != null) {
try {
- dep_groups = prop.getProperty(IntegrityMonitorProperties.DEPENDENCY_GROUPS).split(";");
- if(logger.isDebugEnabled()){
- logger.debug("dependency groups property = {}", Arrays.toString(dep_groups));
+ dep_groups = prop.getProperty(
+ IntegrityMonitorProperties.DEPENDENCY_GROUPS)
+ .split(";");
+ if (logger.isDebugEnabled()) {
+ logger.debug("dependency groups property = {}",
+ Arrays.toString(dep_groups));
}
} catch (Exception e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.DEPENDENCY_GROUPS, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.DEPENDENCY_GROUPS, e);
}
}
-
+
site_name = prop.getProperty(IntegrityMonitorProperties.SITE_NAME);
if (site_name == null) {
- String msg = IntegrityMonitorProperties.SITE_NAME + " property is null";
+ String msg = IntegrityMonitorProperties.SITE_NAME
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
- }else{
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
+ } else {
site_name = site_name.trim();
}
-
+
node_type = prop.getProperty(IntegrityMonitorProperties.NODE_TYPE);
if (node_type == null) {
- String msg = IntegrityMonitorProperties.NODE_TYPE + " property is null";
+ String msg = IntegrityMonitorProperties.NODE_TYPE
+ + " property is null";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
} else {
node_type = node_type.trim();
if (!isNodeTypeEnum(node_type)) {
- String msg = IntegrityMonitorProperties.NODE_TYPE + " property " + node_type + " is invalid";
+ String msg = IntegrityMonitorProperties.NODE_TYPE
+ + " property " + node_type + " is invalid";
logger.error("{}", msg);
- throw new IntegrityMonitorPropertiesException("IntegrityMonitor Property Exception: " + msg);
+ throw new IntegrityMonitorPropertiesException(
+ "IntegrityMonitor Property Exception: " + msg);
}
}
-
+
if (prop.getProperty(IntegrityMonitorProperties.TEST_VIA_JMX) != null) {
- String jmx_test = prop.getProperty(IntegrityMonitorProperties.TEST_VIA_JMX).trim();
+ String jmx_test = prop.getProperty(
+ IntegrityMonitorProperties.TEST_VIA_JMX).trim();
testViaJmx = Boolean.parseBoolean(jmx_test);
}
if (prop.getProperty(IntegrityMonitorProperties.JMX_FQDN) != null) {
- jmxFqdn = prop.getProperty(IntegrityMonitorProperties.JMX_FQDN).trim();
+ jmxFqdn = prop.getProperty(IntegrityMonitorProperties.JMX_FQDN)
+ .trim();
if (jmxFqdn.isEmpty()) {
jmxFqdn = null;
}
}
-
if (prop.getProperty(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL) != null) {
try {
- maxFpcUpdateInterval = Integer.parseInt(prop.getProperty(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL).trim());
+ maxFpcUpdateInterval = Integer.parseInt(prop.getProperty(
+ IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL)
+ .trim());
} catch (NumberFormatException e) {
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, e);
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, e);
}
}
-
- if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null){
- try{
- stateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
- }catch(NumberFormatException e){
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, e);
+
+ if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null) {
+ try {
+ stateAuditIntervalMs = Long
+ .parseLong(prop
+ .getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
+ } catch (NumberFormatException e) {
+ logger.warn("Ignored invalid property: {}",
+ IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, e);
}
}
-
- if (prop.getProperty(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS) != null){
- try{
- refreshStateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS));
- }catch(NumberFormatException e){
- logger.warn("Ignored invalid property: {}", IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, e);
+
+ if (prop.getProperty(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS) != null) {
+ try {
+ refreshStateAuditIntervalMs = Long
+ .parseLong(prop
+ .getProperty(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS));
+ } catch (NumberFormatException e) {
+ logger.warn(
+ "Ignored invalid property: {}",
+ IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS,
+ e);
}
}
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.validateProperties(): Property values \n"
+ + "maxFpcUpdateInterval = {}\n", maxFpcUpdateInterval);
+ }
+
return;
}
-
+
public static void updateProperties(Properties newprop) {
if (isUnitTesting()) {
try {
@@ -1275,14 +1539,13 @@ public class IntegrityMonitor {
} catch (IntegrityMonitorPropertiesException e) {
logger.error("IntegrityMonitor threw exception.", e);
}
- }
- else {
- if(logger.isDebugEnabled()){
+ } else {
+ if (logger.isDebugEnabled()) {
logger.debug("Update integrity monitor properties not allowed");
}
}
}
-
+
private static boolean isNodeTypeEnum(String nodeType) {
for (NodeType n : NodeType.values()) {
if (n.toString().equals(nodeType)) {
@@ -1291,40 +1554,52 @@ public class IntegrityMonitor {
}
return false;
}
-
+
/**
- * Look for "Forward Progress" -- if the 'FPMonitor' is stalled
- * for too long, the operational state is changed to 'Disabled',
- * and an alarm is set. The state is restored when forward
- * progress continues.
+ * Look for "Forward Progress" -- if the 'FPMonitor' is stalled for too
+ * long, the operational state is changed to 'Disabled', and an alarm is
+ * set. The state is restored when forward progress continues.
*/
private void fpMonitorCycle() {
- synchronized(fpMonitorCycleLock){
+ if(logger.isDebugEnabled()){
+ logger.debug("fpMonitorCycle(): entry");
+ }
+ synchronized (fpMonitorCycleLock) {
// monitoring interval checks
if (monitorInterval <= 0) {
+ if(logger.isDebugEnabled()){
+ logger.debug("fpMonitorCycle(): disabled");
+ }
elapsedTime = 0;
return; // monitoring is disabled
}
- elapsedTime = elapsedTime + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
+ elapsedTime = elapsedTime
+ + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
if (elapsedTime < monitorInterval) {
- return; // monitoring interval not reached
+ return; // monitoring interval not reached
}
- elapsedTime = 0; // reset elapsed time
+ elapsedTime = 0; // reset elapsed time
try {
if (fpCounter == lastFpCounter) {
// no forward progress
missedCycles += 1;
if (missedCycles >= failedCounterThreshold && !alarmExists) {
- if(logger.isDebugEnabled()){
- logger.debug("Forward progress not detected for resource {}. Setting state to disable failed.", resourceName);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Forward progress not detected for resource {}. Setting state to disable failed.",
+ resourceName);
}
- if(!(stateManager.getOpState()).equals(StateManagement.DISABLED)){
- // Note: The refreshStateAudit will make redundant calls
+ if (!(stateManager.getOpState())
+ .equals(StateManagement.DISABLED)) {
+ // Note: The refreshStateAudit will make redundant
+ // calls
stateManager.disableFailed();
- }// The refreshStateAudit will catch the case where opStat = disabled and availState ! failed/dependency.failed
+ }// The refreshStateAudit will catch the case where
+ // opStat = disabled and availState !
+ // failed/dependency.failed
alarmExists = true;
}
} else {
@@ -1332,13 +1607,17 @@ public class IntegrityMonitor {
lastFpCounter = fpCounter;
missedCycles = 0;
// set op state to enabled
- if(logger.isDebugEnabled()){
- logger.debug("Forward progress detected for resource {}. Setting state to enable not failed.", resourceName);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Forward progress detected for resource {}. Setting state to enable not failed.",
+ resourceName);
}
- if(!(stateManager.getOpState()).equals(StateManagement.ENABLED)){
+ if (!(stateManager.getOpState())
+ .equals(StateManagement.ENABLED)) {
// Note: The refreshStateAudit will make redundant calls
- stateManager.enableNotFailed();
- }// The refreshStateAudit will catch the case where opState=enabled and availStatus != null
+ stateManager.enableNotFailed();
+ }// The refreshStateAudit will catch the case where
+ // opState=enabled and availStatus != null
alarmExists = false;
}
} catch (Exception e) {
@@ -1346,28 +1625,46 @@ public class IntegrityMonitor {
logger.error("FP Monitor encountered error. ", e);
}
}
+ if(logger.isDebugEnabled()){
+ logger.debug("fpMonitorCycle(): exit");
+ }
}
-
+
/**
- * Look for "Forward Progress" on other nodes. If they are not making forward progress,
- * check their operational state. If it is not disabled, then disable them.
+ * Look for "Forward Progress" on other nodes. If they are not making
+ * forward progress, check their operational state. If it is not disabled,
+ * then disable them.
*/
- public void stateAudit() {
-
+ private void stateAudit() {
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): entry");
+ }
if (stateAuditIntervalMs <= 0) {
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): disabled");
+ }
return; // stateAudit is disabled
}
//Only run from nodes that are operational
if(stateManager.getOpState().equals(StateManagement.DISABLED)){
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): DISABLED. returning");
+ }
return;
}
if(stateManager.getAdminState().equals(StateManagement.LOCKED)){
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): LOCKED. returning");
+ }
return;
}
if(!stateManager.getStandbyStatus().equals(StateManagement.NULL_VALUE) &&
stateManager.getStandbyStatus()!= null){
if(!stateManager.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE)){
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): NOT PROVIDING_SERVICE. returning");
+ }
return;
}
}
@@ -1375,8 +1672,26 @@ public class IntegrityMonitor {
Date date = new Date();
long timeSinceLastStateAudit = date.getTime() - lastStateAuditTime.getTime();
if (timeSinceLastStateAudit < stateAuditIntervalMs){
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): Not time to run. returning");
+ }
return;
}
+
+ executeStateAudit();
+
+ lastStateAuditTime = date;
+
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.stateAudit(): exit");
+ }
+ }// end stateAudit()
+
+ public void executeStateAudit(){
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.executeStateAudit(): entry");
+ }
+ Date date = new Date();
// Get all entries in the forwardprogressentity table
ArrayList<ForwardProgressEntity> fpList = getAllForwardProgressEntity();
@@ -1391,18 +1706,21 @@ public class IntegrityMonitor {
em.refresh(fpe);
long diffMs = date.getTime() - fpe.getLastUpdated().getTime();
if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateAudit(): diffMs = {}", diffMs);
+ logger.debug("IntegrityMonitor.executeStateAudit(): resource = {}, diffMs = {}", fpe.getResourceName(), diffMs);
}
//Threshold for a stale entry
- long staleMs = maxFpcUpdateInterval * (long)1000;
+ long staleMs = 1000L * maxFpcUpdateInterval;
if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateAudit(): staleMs = {}", staleMs);
+ logger.debug("IntegrityMonitor.executeStateAudit(): resource = {}, staleMs = {}", fpe.getResourceName(), staleMs);
}
if(diffMs > staleMs){
//ForwardProgress is stale. Disable it
// Start a transaction
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.executeStateAudit(): resource = {}, FPC is stale. Disabling it" );
+ }
EntityTransaction et = em.getTransaction();
et.begin();
StateManagementEntity sme = null;
@@ -1420,10 +1738,10 @@ public class IntegrityMonitor {
// refresh the object from DB in case cached data was returned
em.refresh(sme);
if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateAudit(): Found entry in StateManagementEntity table for Resource={}", sme.getResourceName());
+ logger.debug("IntegrityMonitor.executeStateAudit(): Found entry in StateManagementEntity table for Resource={}", sme.getResourceName());
}
} else {
- String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
+ String msg = "IntegrityMonitor.executeStateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
logger.error("{}", msg);
}
synchronized(IMFLUSHLOCK){
@@ -1431,7 +1749,7 @@ public class IntegrityMonitor {
}
} catch (Exception e) {
// log an error
- logger.error("IntegrityMonitor.stateAudit(): {}: StateManagementEntity DB read failed with exception: ", fpe.getResourceName(), e);
+ logger.error("IntegrityMonitor.executeStateAudit(): {}: StateManagementEntity DB read failed with exception: ", fpe.getResourceName(), e);
synchronized(IMFLUSHLOCK){
if(et.isActive()){
et.rollback();
@@ -1441,62 +1759,81 @@ public class IntegrityMonitor {
if(sme != null && !sme.getOpState().equals(StateManagement.DISABLED)){
if(logger.isDebugEnabled()){
- logger.debug("IntegrityMonitor.stateAudit(): Changing OpStat = disabled for {}", sme.getResourceName());
+ logger.debug("IntegrityMonitor.executeStateAudit(): Changing OpStat = disabled for {}", sme.getResourceName());
}
try {
stateManager.disableFailed(sme.getResourceName());
} catch (Exception e) {
- String msg = "IntegrityMonitor.stateAudit(): Failed to disable " + sme.getResourceName();
+ String msg = "IntegrityMonitor.executeStateAudit(): Failed to disable " + sme.getResourceName();
logger.error("{}", msg, e);
}
}
}// end if(diffMs > staleMs)
}// end for(ForwardProgressEntity fpe : fpList)
- lastStateAuditTime = date;
- }// end stateAudit()
+ if(logger.isDebugEnabled()){
+ logger.debug("IntegrityMonitor.executeStateAudit(): exit");
+ }
+ }
/**
* Execute a test transaction when test transaction interval has elapsed.
*/
private void checkTestTransaction() {
- synchronized(checkTestTransactionLock){
+ if(logger.isDebugEnabled()){
+ logger.debug("checkTestTransaction(): entry");
+ }
+ synchronized (checkTestTransactionLock) {
// test transaction timer checks
if (testTransInterval <= 0) {
+ if(logger.isDebugEnabled()){
+ logger.debug("checkTestTransaction(): disabled");
+ }
elapsedTestTransTime = 0;
return; // test transaction is disabled
}
- elapsedTestTransTime = elapsedTestTransTime + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
+ elapsedTestTransTime = elapsedTestTransTime
+ + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
if (elapsedTestTransTime < testTransInterval) {
- return; // test transaction interval not reached
+ return; // test transaction interval not reached
}
- elapsedTestTransTime = 0; // reset elapsed time
+ elapsedTestTransTime = 0; // reset elapsed time
// execute test transaction
testTransaction();
}
+ if(logger.isDebugEnabled()){
+ logger.debug("checkTestTransaction(): exit");
+ }
}
-
+
/**
* Updates Fpc counter in database when write Fpc interval has elapsed.
*/
private void checkWriteFpc() {
- synchronized(checkWriteFpcLock){
+ if(logger.isDebugEnabled()){
+ logger.debug("checkWriteFpc(): entry");
+ }
+ synchronized (checkWriteFpcLock) {
// test transaction timer checks
if (writeFpcInterval <= 0) {
+ if(logger.isDebugEnabled()){
+ logger.debug("checkWriteFpc(): disabled");
+ }
elapsedWriteFpcTime = 0;
return; // write Fpc is disabled
}
- elapsedWriteFpcTime = elapsedWriteFpcTime + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
+ elapsedWriteFpcTime = elapsedWriteFpcTime
+ + TimeUnit.MILLISECONDS.toSeconds(CYCLE_INTERVAL_MILLIS);
if (elapsedWriteFpcTime < writeFpcInterval) {
- return; // write Fpc interval not reached
+ return; // write Fpc interval not reached
}
- elapsedWriteFpcTime = 0; // reset elapsed time
+ elapsedWriteFpcTime = 0; // reset elapsed time
// write Fpc to database
try {
@@ -1505,84 +1842,126 @@ public class IntegrityMonitor {
logger.error("IntegrityMonitor threw exception.", e);
}
}
+ if(logger.isDebugEnabled()){
+ logger.debug("checkWriteFpc(): exit");
+ }
}
-
+
/**
- * Execute a dependency health check periodically which also updates this resource's state.
+ * Execute a dependency health check periodically which also updates this
+ * resource's state.
*/
private void checkDependentHealth() {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("checkDependentHealth: entry");
}
-
+ if (checkDependencyInterval <=0) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("checkDependentHealth: disabled");
+ }
+ return; // dependency monitoring is disabled
+ }
+
long currTime = System.currentTimeMillis();
- if(logger.isDebugEnabled()){
- logger.debug("checkDependentHealth currTime - lastDependencyCheckTime = {}", (currTime - lastDependencyCheckTime));
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "checkDependentHealth currTime - lastDependencyCheckTime = {}",
+ (currTime - lastDependencyCheckTime));
}
- if ((currTime - lastDependencyCheckTime) > (1000 * IntegrityMonitorProperties.DEFAULT_TEST_INTERVAL)) {
+ if ((currTime - lastDependencyCheckTime) > (1000 * checkDependencyInterval)) {
// execute dependency check and update this resource's state
-
+
dependencyCheck();
}
+ if (logger.isDebugEnabled()) {
+ logger.debug("checkDependentHealth: exit");
+ }
}
-
+
/*
- * This is a simple refresh audit which is periodically run to assure that the states and status
- * attributes are aligned and notifications are sent to any listeners. It is possible for state/status
- * to get out of synch and notified systems to be out of synch due to database corruption (manual or
+ * This is a simple refresh audit which is periodically run to assure that
+ * the states and status attributes are aligned and notifications are sent
+ * to any listeners. It is possible for state/status to get out of synch and
+ * notified systems to be out of synch due to database corruption (manual or
* otherwise) or because a node became isolated.
*
- * When the operation (lock/unlock) is called, it will cause a re-evaluation of the state and
- * send a notification to all registered observers.
+ * When the operation (lock/unlock) is called, it will cause a re-evaluation
+ * of the state and send a notification to all registered observers.
*/
- private void refreshStateAudit(){
- if(refreshStateAuditIntervalMs <=0){
- // The audit is deactivated
- return;
+ private void refreshStateAudit() {
+ if(logger.isDebugEnabled()){
+ logger.debug("refreshStateAudit(): entry");
}
- synchronized(refreshStateAuditLock){
+ if (refreshStateAuditIntervalMs <= 0) {
+ // The audit is disabled
if(logger.isDebugEnabled()){
+ logger.debug("refreshStateAudit(): disabled");
+ }
+ return;
+ }
+ executeRefreshStateAudit();
+ if(logger.isDebugEnabled()){
+ logger.debug("refreshStateAudit(): exit");
+ }
+ }
+
+ public void executeRefreshStateAudit(){
+ if(logger.isDebugEnabled()){
+ logger.debug("executeRefreshStateAudit(): entry");
+ }
+ synchronized (refreshStateAuditLock) {
+ if (logger.isDebugEnabled()) {
logger.debug("refreshStateAudit: entry");
}
Date now = new Date();
long nowMs = now.getTime();
long lastTimeMs = refreshStateAuditLastRunDate.getTime();
- if(logger.isDebugEnabled()){
- logger.debug("refreshStateAudit: ms since last run = {}", (nowMs - lastTimeMs));
+ if (logger.isDebugEnabled()) {
+ logger.debug("refreshStateAudit: ms since last run = {}",
+ (nowMs - lastTimeMs));
}
- if((nowMs - lastTimeMs) > refreshStateAuditIntervalMs){
+ if ((nowMs - lastTimeMs) > refreshStateAuditIntervalMs) {
String adminState = stateManager.getAdminState();
- if(logger.isDebugEnabled()){
- logger.debug("refreshStateAudit: adminState = {}", adminState);
+ if (logger.isDebugEnabled()) {
+ logger.debug("refreshStateAudit: adminState = {}",
+ adminState);
}
- if(adminState.equals(StateManagement.LOCKED)){
+ if (adminState.equals(StateManagement.LOCKED)) {
try {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("refreshStateAudit: calling lock()");
}
stateManager.lock();
} catch (Exception e) {
- logger.error("refreshStateAudit: caught unexpected exception from stateManager.lock(): ", e);
+ logger.error(
+ "refreshStateAudit: caught unexpected exception from stateManager.lock(): ",
+ e);
}
- }else{//unlocked
+ } else {// unlocked
try {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("refreshStateAudit: calling unlock()");
}
- stateManager.unlock();;
+ stateManager.unlock();
+ ;
} catch (Exception e) {
- logger.error("refreshStateAudit: caught unexpected exception from stateManager.unlock(): ", e);
+ logger.error(
+ "refreshStateAudit: caught unexpected exception from stateManager.unlock(): ",
+ e);
}
}
refreshStateAuditLastRunDate = new Date();
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("refreshStateAudit: exit");
}
}
}
+ if(logger.isDebugEnabled()){
+ logger.debug("executeRefreshStateAudit(): exit");
+ }
}
-
+
public static boolean isUnitTesting() {
return isUnitTesting;
}
@@ -1592,11 +1971,13 @@ public class IntegrityMonitor {
}
/**
- * The following nested class periodically performs the forward progress check,
- * checks dependencies, does a refresh state audit and runs the stateAudit.
+ * The following nested class periodically performs the forward progress
+ * check, checks dependencies, does a refresh state audit and runs the
+ * stateAudit.
*/
class FPManager extends Thread {
-
+ private volatile boolean exit = false;
+
// Constructor - start FP manager thread
FPManager() {
// set now as the last time the refreshStateAudit ran
@@ -1604,68 +1985,161 @@ public class IntegrityMonitor {
// start thread
this.start();
}
-
+
@Override
public void run() {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager thread running");
}
- while (true) {
+ while (!exit) {
try {
Thread.sleep(CYCLE_INTERVAL_MILLIS);
} catch (InterruptedException e) {
// The 'sleep' call was interrupted
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("IntegrityMonitor threw exception.", e);
}
Thread.currentThread().interrupt();
continue;
}
-
+
try {
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling fpMonitorCycle()");
}
// check forward progress timer
IntegrityMonitor.this.fpMonitorCycle();
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling checkTestTransaction()");
}
// check test transaction timer
IntegrityMonitor.this.checkTestTransaction();
- if(logger.isDebugEnabled()){
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling checkWriteFpc()");
}
// check write Fpc timer
IntegrityMonitor.this.checkWriteFpc();
-
- if(logger.isDebugEnabled()){
+
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling checkDependentHealth()");
}
// check dependency health
IntegrityMonitor.this.checkDependentHealth();
-
- if(logger.isDebugEnabled()){
+
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling refreshStateAudit()");
}
// check if it is time to run the refreshStateAudit
IntegrityMonitor.this.refreshStateAudit();
-
- if(logger.isDebugEnabled()){
+
+ if (logger.isDebugEnabled()) {
logger.debug("FPManager calling stateAudit()");
}
// check if it is time to run the stateAudit
IntegrityMonitor.this.stateAudit();
-
+
} catch (Exception e) {
- logger.error("Ignore FPManager thread processing timer(s) exception: ", e);
+ logger.error(
+ "Ignore FPManager thread processing timer(s) exception: ",
+ e);
}
}
}
+
+ public void stopAndExit() {
+ exit = true;
+ // Referring to the FPManager instance
+ FPManager.this.interrupt();
+ }
+
+ }
+
+ public void allSeemsWell(@NotNull String key, @NotNull Boolean asw, @NotNull String msg)
+ throws IllegalArgumentException, AllSeemsWellException {
+
+ if(logger.isDebugEnabled()){
+ logger.debug("allSeemsWell entry: key = {}, asw = {}, msg = {}", key, asw, msg);
+ }
+ if(key == null || key.isEmpty()){
+ logger.error("allSeemsWell: 'key' has no visible content");
+ throw new IllegalArgumentException("allSeemsWell: 'key' has no visible content");
+ }
+ if(asw == null){
+ logger.error("allSeemsWell: 'asw' is null");
+ throw new IllegalArgumentException("allSeemsWell: 'asw' is null");
+ }
+ if(msg == null || msg.isEmpty()){
+ logger.error("allSeemsWell: 'msg' has no visible content");
+ throw new IllegalArgumentException("allSeemsWell: 'msg' has no visible content");
+ }
+
+ if(allSeemsWellMap == null){
+ allSeemsWellMap = new HashMap<>();
+ }
+ if(allNotWellMap == null){
+ allNotWellMap = new HashMap<>();
+ }
+
+ if(asw){
+ logger.info("allSeemsWell: ALL SEEMS WELL: key = {}, msg = {}", key, msg);
+ try{
+ allSeemsWellMap.put(key, msg);
+ }catch(Exception e){
+ String exceptMsg ="allSeemsWell: encountered an exception with allSeemsWellMap.put("
+ + key + "," + msg + ")";
+ logger.error(exceptMsg);
+ throw new AllSeemsWellException(exceptMsg, e);
+ }
+
+ try{
+ allNotWellMap.remove(key);
+ }catch(Exception e){
+ String exceptMsg ="allSeemsWell: encountered an exception with allNotWellMap.delete("
+ + key + ")";
+ logger.error(exceptMsg);
+ throw new AllSeemsWellException(exceptMsg, e);
+ }
+
+ }else{
+ logger.error("allSeemsWell: ALL NOT WELL: key = {}, msg = {}", key, msg);
+ try{
+ allSeemsWellMap.remove(key);
+ }catch(Exception e){
+ String exceptMsg ="allSeemsWell: encountered an exception with allSeemsWellMap.remove("
+ + key + ")";
+ logger.error(exceptMsg);
+ throw new AllSeemsWellException(exceptMsg, e);
+ }
+
+ try{
+ allNotWellMap.put(key, msg);
+ }catch(Exception e){
+ String exceptMsg ="allSeemsWell: encountered an exception with allNotWellMap.put("
+ + key + msg + ")";
+ logger.error(exceptMsg);
+ throw new AllSeemsWellException(exceptMsg, e);
+ }
+ }
+
+ if(logger.isDebugEnabled()){
+ for(String mykey: allSeemsWellMap.keySet()){
+ logger.debug("allSeemsWellMap: key = {} msg = {}", mykey, allSeemsWellMap.get(mykey));
+ }
+ for(String mykey: allNotWellMap.keySet()){
+ logger.debug("allNotWellMap: key = {} msg = {}", mykey, allNotWellMap.get(mykey));
+ }
+ logger.debug("allSeemsWell exit");
+ }
}
-}
+ public Map<String, String> getAllSeemsWellMap(){
+ return allSeemsWellMap;
+ }
+ public Map<String, String> getAllNotWellMap(){
+ return allNotWellMap;
+ }
+}
diff --git a/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitorProperties.java b/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitorProperties.java
index 74eace68..b495db71 100644
--- a/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitorProperties.java
+++ b/integrity-monitor/src/main/java/org/onap/policy/common/im/IntegrityMonitorProperties.java
@@ -35,11 +35,13 @@ public class IntegrityMonitorProperties {
public static final int DEFAULT_TEST_INTERVAL = 10;
public static final int DEFAULT_WRITE_FPC_INTERVAL = 5;
public static final int DEFAULT_MAX_FPC_UPDATE_INTERVAL = 120;
+ public static final int DEFAULT_CHECK_DEPENDENCY_INTERVAL = 10;
public static final String FP_MONITOR_INTERVAL = "fp_monitor_interval";
public static final String FAILED_COUNTER_THRESHOLD = "failed_counter_threshold";
public static final String TEST_TRANS_INTERVAL = "test_trans_interval";
public static final String WRITE_FPC_INTERVAL = "write_fpc_interval";
+ public static final String CHECK_DEPENDENCY_INTERVAL = "check_dependency_interval";
public static final String DEPENDENCY_GROUPS = "dependency_groups";
public static final String SITE_NAME = "site_name";
@@ -51,4 +53,9 @@ public class IntegrityMonitorProperties {
public static final String STATE_AUDIT_INTERVAL_MS = "state_audit_interval_ms";
public static final String REFRESH_STATE_AUDIT_INTERVAL_MS = "refresh_state_audit_interval_ms";
+ // AllSeemsWell types
+ public static final Boolean ALLNOTWELL=Boolean.FALSE;
+ public static final Boolean ALLSEEMSWELL=Boolean.TRUE;
+
+
}
diff --git a/integrity-monitor/src/test/java/org/onap/policy/common/im/test/AllSeemsWellTest.java b/integrity-monitor/src/test/java/org/onap/policy/common/im/test/AllSeemsWellTest.java
new file mode 100644
index 00000000..6fa8114d
--- /dev/null
+++ b/integrity-monitor/src/test/java/org/onap/policy/common/im/test/AllSeemsWellTest.java
@@ -0,0 +1,265 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * Integrity Monitor
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.common.im.test;
+
+import static org.junit.Assert.*;
+
+import java.util.Map;
+import java.util.Properties;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.EntityTransaction;
+import javax.persistence.Persistence;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onap.policy.common.im.IntegrityMonitor;
+import org.onap.policy.common.im.IntegrityMonitorProperties;
+import org.onap.policy.common.im.StateManagement;
+import org.onap.policy.common.im.jpa.ForwardProgressEntity;
+import org.onap.policy.common.im.jpa.StateManagementEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AllSeemsWellTest {
+ private static Logger logger = LoggerFactory.getLogger(AllSeemsWellTest.class);
+ private static Properties myProp;
+ private static EntityManagerFactory emf;
+ private static EntityManager em;
+ private static EntityTransaction et;
+ private static String resourceName;
+ private static Properties systemProps;
+
+ private static final String DEFAULT_DB_DRIVER = "org.h2.Driver";
+ private static final String DEFAULT_DB_URL = "jdbc:h2:file:./sql/imTest";
+ private static final String DEFAULT_DB_USER = "sa";
+ private static final String DEFAULT_DB_PWD = "";
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ IntegrityMonitor.setUnitTesting(true);
+
+ myProp = new Properties();
+ myProp.put(IntegrityMonitorProperties.DB_DRIVER, AllSeemsWellTest.DEFAULT_DB_DRIVER);
+ myProp.put(IntegrityMonitorProperties.DB_URL, AllSeemsWellTest.DEFAULT_DB_URL);
+ myProp.put(IntegrityMonitorProperties.DB_USER, AllSeemsWellTest.DEFAULT_DB_USER);
+ myProp.put(IntegrityMonitorProperties.DB_PWD, AllSeemsWellTest.DEFAULT_DB_PWD);
+ myProp.put(IntegrityMonitorProperties.SITE_NAME, "SiteA");
+ myProp.put(IntegrityMonitorProperties.NODE_TYPE, "pap");
+
+ // set JMX remote port in system properties
+ systemProps = System.getProperties();
+ systemProps.put("com.sun.management.jmxremote.port", "9797");
+
+ resourceName = "siteA.pap1";
+
+ //Create the data schema and entity manager factory
+ emf = Persistence.createEntityManagerFactory("schemaPU", myProp);
+
+ // Create an entity manager to use the DB
+ em = emf.createEntityManager();
+
+ }
+
+
+ @After
+ public void tearDown() throws Exception {
+ // clear jmx remote port setting
+ systemProps.remove("com.sun.management.jmxremote.port");
+ }
+
+ //Ignore
+ @Test
+ public void testAllSeemsWell() throws Exception {
+ logger.debug("\nIntegrityMonitorTest: Entering testAllSeemsWell\n\n");
+
+ // parameters are passed via a properties file
+ myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+ myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, "1");
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "5");
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "1");
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "1");
+
+ IntegrityMonitor.updateProperties(myProp);
+ /*
+ * The monitorInterval is 5 and the failedCounterThreshold is 1
+ * A forward progress will be stale after 5 seconds.
+ */
+
+ et = em.getTransaction();
+ et.begin();
+
+ // Make sure we start with the DB clean
+ em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+ em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+ em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+ em.flush();
+ et.commit();
+
+ IntegrityMonitor.deleteInstance();
+
+ IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+
+ StateManagement sm = im.getStateManager();
+
+ //Give it time to set the states in the DB
+ Thread.sleep(15000);
+
+ //Check the state
+ logger.debug("\n\ntestAllSeemsWell starting im state \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+ sm.getAdminState(),
+ sm.getOpState(),
+ sm.getAvailStatus(),
+ sm.getStandbyStatus());
+
+ assertTrue(sm.getOpState().equals(StateManagement.ENABLED));
+
+ //Indicate a failure
+ im.allSeemsWell(this.getClass().getName(), IntegrityMonitorProperties.ALLNOTWELL,
+ "'AllSeemsWellTest - ALLNOTWELL'");
+
+ //Wait for the state to change due to ALLNOTWELL
+ Thread.sleep(15000);
+ //Check the state
+ logger.debug("\n\ntestAllSeemsWell after ALLNOTWELL: im state \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+ sm.getAdminState(),
+ sm.getOpState(),
+ sm.getAvailStatus(),
+ sm.getStandbyStatus());
+
+ //assertTrue(sm.getOpState().equals(StateManagement.DISABLED));
+
+ Map<String, String> allNotWellMap = im.getAllNotWellMap();
+ for(String key: allNotWellMap.keySet()){
+ logger.debug("AllSeemsWellTest: allNotWellMap: key = {} msg = {}", key, allNotWellMap.get(key));
+ }
+ //assertTrue(allNotWellMap.size() == 1);
+
+ Map<String,String> allSeemsWellMap = im.getAllSeemsWellMap();
+ //assertTrue(allSeemsWellMap.isEmpty());
+
+ //Return to normal
+ im.allSeemsWell(this.getClass().getName(), IntegrityMonitorProperties.ALLSEEMSWELL,
+ "'AllSeemsWellTest - ALLSEEMSWELL'");
+
+ //Wait for the state to change due to ALLNOTWELL
+ Thread.sleep(15000);
+ //Check the state
+ logger.debug("\n\ntestAllSeemsWell after ALLSEEMSWELL: im state \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+ sm.getAdminState(),
+ sm.getOpState(),
+ sm.getAvailStatus(),
+ sm.getStandbyStatus());
+
+ //assertTrue(sm.getOpState().equals(StateManagement.ENABLED));
+
+ allNotWellMap = im.getAllNotWellMap();
+ assertTrue(allNotWellMap.isEmpty());
+
+ allSeemsWellMap = im.getAllSeemsWellMap();
+ assertTrue(allSeemsWellMap.size() == 1);
+ for(String key: allSeemsWellMap.keySet()){
+ logger.debug("AllSeemsWellTest: allSeemsWellMap: key = {} msg = {}", key, allSeemsWellMap.get(key));
+ }
+
+ //Check for null parameters
+ try{
+ im.allSeemsWell(null, IntegrityMonitorProperties.ALLSEEMSWELL,
+ "'AllSeemsWellTest - ALLSEEMSWELL'");
+ assertTrue(false);
+ }catch (IllegalArgumentException e) {
+ assertTrue(true);
+ }
+
+ try{
+ im.allSeemsWell("", IntegrityMonitorProperties.ALLSEEMSWELL,
+ "'AllSeemsWellTest - ALLSEEMSWELL'");
+ assertTrue(false);
+ }catch (IllegalArgumentException e) {
+ assertTrue(true);
+ }
+
+ try{
+ im.allSeemsWell(this.getClass().getName(), null,
+ "'AllSeemsWellTest - ALLSEEMSWELL'");
+ assertTrue(false);
+ }catch (IllegalArgumentException e) {
+ assertTrue(true);
+ }
+
+ try{
+ im.allSeemsWell(this.getClass().getName(), IntegrityMonitorProperties.ALLSEEMSWELL,
+ null);
+ assertTrue(false);
+ }catch (IllegalArgumentException e) {
+ assertTrue(true);
+ }
+
+ try{
+ im.allSeemsWell(this.getClass().getName(), IntegrityMonitorProperties.ALLSEEMSWELL,
+ "");
+ assertTrue(false);
+ }catch (IllegalArgumentException e) {
+ assertTrue(true);
+ }
+
+ // undo settings
+ myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, Integer.toString(IntegrityMonitorProperties.DEFAULT_FAILED_COUNTER_THRESHOLD));
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_MONITOR_INTERVAL));
+ myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, Integer.toString(IntegrityMonitorProperties.DEFAULT_FAILED_COUNTER_THRESHOLD));
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_MONITOR_INTERVAL));
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_TEST_INTERVAL));
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_WRITE_FPC_INTERVAL));
+ IntegrityMonitor.updateProperties(myProp);
+
+ et = em.getTransaction();
+
+ et.begin();
+ // Make sure we leave the DB clean
+ em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+ em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+ em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+ em.flush();
+ et.commit();
+
+ logger.debug("\n\ntestAllSeemsWell: Exit\n\n");
+ }
+
+}
diff --git a/integrity-monitor/src/test/java/org/onap/policy/common/im/test/IntegrityMonitorTest.java b/integrity-monitor/src/test/java/org/onap/policy/common/im/test/IntegrityMonitorTest.java
index de91a91d..563c5d55 100644
--- a/integrity-monitor/src/test/java/org/onap/policy/common/im/test/IntegrityMonitorTest.java
+++ b/integrity-monitor/src/test/java/org/onap/policy/common/im/test/IntegrityMonitorTest.java
@@ -38,8 +38,8 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
-
import org.onap.policy.common.im.IntegrityMonitor;
import org.onap.policy.common.im.IntegrityMonitorProperties;
import org.onap.policy.common.im.StateManagement;
@@ -81,13 +81,7 @@ public class IntegrityMonitorTest {
public void setUp() throws Exception {
IntegrityMonitor.setUnitTesting(true);
- myProp = new Properties();
- myProp.put(IntegrityMonitorProperties.DB_DRIVER, IntegrityMonitorTest.DEFAULT_DB_DRIVER);
- myProp.put(IntegrityMonitorProperties.DB_URL, IntegrityMonitorTest.DEFAULT_DB_URL);
- myProp.put(IntegrityMonitorProperties.DB_USER, IntegrityMonitorTest.DEFAULT_DB_USER);
- myProp.put(IntegrityMonitorProperties.DB_PWD, IntegrityMonitorTest.DEFAULT_DB_PWD);
- myProp.put(IntegrityMonitorProperties.SITE_NAME, "SiteA");
- myProp.put(IntegrityMonitorProperties.NODE_TYPE, "pap");
+ cleanMyProp();
// set JMX remote port in system properties
systemProps = System.getProperties();
@@ -110,21 +104,44 @@ public class IntegrityMonitorTest {
systemProps.remove("com.sun.management.jmxremote.port");
}
+ private void cleanMyProp(){
+ myProp = new Properties();
+ myProp.put(IntegrityMonitorProperties.DB_DRIVER, IntegrityMonitorTest.DEFAULT_DB_DRIVER);
+ myProp.put(IntegrityMonitorProperties.DB_URL, IntegrityMonitorTest.DEFAULT_DB_URL);
+ myProp.put(IntegrityMonitorProperties.DB_USER, IntegrityMonitorTest.DEFAULT_DB_USER);
+ myProp.put(IntegrityMonitorProperties.DB_PWD, IntegrityMonitorTest.DEFAULT_DB_PWD);
+ myProp.put(IntegrityMonitorProperties.SITE_NAME, "SiteA");
+ myProp.put(IntegrityMonitorProperties.NODE_TYPE, "pap");
+ }
+
+ private void cleanDb(){
+ et = em.getTransaction();
+
+ et.begin();
+ // Make sure we leave the DB clean
+ em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+ em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+ em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+ em.flush();
+ et.commit();
+ }
+
+
/*
* The following runs all tests and controls the order of execution. If you allow
* the tests to execute individually, you cannot predict the order and some
- * conflicts occur.
+ * conflicts may occur.
*/
//@Ignore
@Test
public void runAllTests() throws Exception{
testSanityJmx();
- testIM();
- //testSanityState();
- //testRefreshStateAudit();
+ testIM();
+ testSanityState();
+ testRefreshStateAudit();
testStateCheck();
- //testGetAllForwardProgressEntity();
- testStateAudit();
+ testGetAllForwardProgressEntity();
+ testStateAudit();
}
/*
@@ -139,13 +156,29 @@ public class IntegrityMonitorTest {
*/
public void testSanityJmx() throws Exception {
logger.debug("\nIntegrityMonitorTest: Entering testSanityJmx\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
String dependent = "group1_logparser";
// parameters are passed via a properties file
myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, dependent);
myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "true");
- IntegrityMonitor.updateProperties(myProp);
+ // Disable the integrity monitor so it will not interfere
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the state audit
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable the write FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+ // Speed up the check
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "1");
+ // Fail dependencies after three seconds
+ myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "3");
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
logger.debug("\n\ntestSanityJmx starting im state \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
@@ -181,7 +214,7 @@ public class IntegrityMonitorTest {
// commit transaction
et.commit();
- Thread.sleep(15000); //sleep 15 sec so the FPManager has time to call evaluateSanty()
+ Thread.sleep(5000); //sleep 5 sec so the FPManager has time to check dependency health
boolean sanityPass = true;
try {
@@ -287,26 +320,26 @@ public class IntegrityMonitorTest {
public void testIM() throws Exception {
logger.debug("\nIntegrityMonitorTest: Entering testIM\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
- // parameters are passed via a properties file
+ // Disable the integrity monitor so it will not interfere
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable dependency checking
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the state audit
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable writing the FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
- /*
- * Create an IntegrityMonitor
- * NOTE: This uses the database that was created above. So, this MUST follow the creation
- * of the DB
- */
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
- logger.debug("\n\nim before sleep\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
- im.getStateManager().getAdminState(),
- im.getStateManager().getOpState(),
- im.getStateManager().getAvailStatus(),
- im.getStateManager().getStandbyStatus());
-
- // wait for test transactions to fire and increment fpc
- Thread.sleep(20000);
-
- logger.debug("\n\nim after sleep\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+ logger.debug("\n\nim initial state: \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
im.getStateManager().getAdminState(),
im.getStateManager().getOpState(),
im.getStateManager().getAvailStatus(),
@@ -336,6 +369,7 @@ public class IntegrityMonitorTest {
StateManagement sm = im.getStateManager();
sm.lock();
+
logger.debug("\n\nsm.lock()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
sm.getAdminState(),
sm.getOpState(),
@@ -573,10 +607,26 @@ public class IntegrityMonitorTest {
public void testSanityState() throws Exception {
logger.debug("\nIntegrityMonitorTest: Entering testSanityState\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
// parameters are passed via a properties file
myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "group1_dep1,group1_dep2; group2_dep1");
- IntegrityMonitor.updateProperties(myProp);
+ // Disable the integrity monitor so it will not interfere
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable dependency checking so it does not interfere
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+ // Disable the state audit
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable writing the FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+ // Max interval for use in deciding if a FPC entry is stale in seconds
+ myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "120");
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
@@ -604,7 +654,9 @@ public class IntegrityMonitorTest {
new StateManagement(emf, "group1_dep2");
boolean sanityPass = true;
- Thread.sleep(15000);
+ // Call the dependency check directly instead of waiting for FPManager to do it.
+ logger.debug("\n\nIntegrityMonitor.testSanityState: calling im.dependencyCheck()\n\n");
+ im.dependencyCheck();
try {
im.evaluateSanity();
} catch (Exception e) {
@@ -613,48 +665,32 @@ public class IntegrityMonitorTest {
}
assertFalse(sanityPass); // expect sanity test to fail
- // undo dependency groups and jmx test properties settings
- myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
- myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
- IntegrityMonitor.updateProperties(myProp);
-
- et = em.getTransaction();
-
- et.begin();
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
logger.debug("\n\ntestSanityState: Exit\n\n");
}
public void testRefreshStateAudit() throws Exception {
logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Enter\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
// parameters are passed via a properties file
myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
- myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "60000");
- IntegrityMonitor.updateProperties(myProp);
+ // Disable the integrity monitor so it will not interfere
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable dependency checking so it does not interfere
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+ // Disable the state audit
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable writing the FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
- et = em.getTransaction();
- et.begin();
-
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
- IntegrityMonitor.deleteInstance();
-
- IntegrityMonitor.getInstance(resourceName, myProp);
+ IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
//the state here is unlocked, enabled, null, null
StateManagementEntity sme = null;
@@ -697,7 +733,8 @@ public class IntegrityMonitorTest {
em.flush();
et.commit();
- Thread.sleep(65000);
+ // Run the refreshStateAudit
+ im.executeRefreshStateAudit();
//The refreshStateAudit should run and change the state to unlocked,enabled,null,hotstandby
StateManagementEntity sme1 = null;
@@ -706,7 +743,6 @@ public class IntegrityMonitorTest {
query1.setParameter("resource", resourceName);
- //Just test that we are retrieving the right object
@SuppressWarnings("rawtypes")
List resourceList1 = query1.getResultList();
if (!resourceList1.isEmpty()) {
@@ -731,49 +767,36 @@ public class IntegrityMonitorTest {
assertTrue(false);
}
- et = em.getTransaction();
- et.begin();
-
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
- IntegrityMonitor.deleteInstance();
-
logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Exit\n\n");
}
public void testStateCheck() throws Exception {
logger.debug("\nIntegrityMonitorTest: Entering testStateCheck\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
// parameters are passed via a properties file
myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "group1_dep1");
myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, "1");
- myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "10");
- IntegrityMonitor.updateProperties(myProp);
/*
- * The default monitorInterval is 30 and the default failedCounterThreshold is 3
- * Since stateCheck() uses the faileCounterThreshold * monitorInterval to determine
- * if an entry is stale, it will be stale after 30 seconds.
+ * The monitorInterval is set to 10 and the failedCounterThreshold is 1
+ * because stateCheck() uses the faileCounterThreshold * monitorInterval to determine
+ * if an entry is stale, it will be stale after 10 seconds.
*/
-
- et = em.getTransaction();
- et.begin();
-
- // Make sure we start with the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
- IntegrityMonitor.deleteInstance();
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "5");
+ /*
+ * We accelerate the test transaction and write FPC intervals because we don't want
+ * there to be any chance of a FPC failure because of the short monitor interval
+ */
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "1");
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "2");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // The maximum time in seconds to determine that a FPC entry is stale
+ myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "5");
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "5");
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
@@ -792,73 +815,56 @@ public class IntegrityMonitorTest {
boolean sanityPass = true;
//Thread.sleep(15000);
- Thread.sleep(5000);
+ //Thread.sleep(5000);
try {
im.evaluateSanity();
} catch (Exception e) {
- logger.error("testStateCheck: After 15 sec sleep - evaluateSanity exception: ", e);
+ logger.error("testStateCheck: After 5 sec sleep - evaluateSanity exception: ", e);
sanityPass = false;
}
assertTrue(sanityPass); // expect sanity test to pass
- //now wait 30 seconds. The dependency entry should now be stale and the sanitry check should fail
+ //now wait 10 seconds. The dependency entry is checked every 10 sec. So, even in the worst case
+ //it should now be stale and the sanity check should fail
sanityPass = true;
- //Thread.sleep(30000);
Thread.sleep(10000);
try {
im.evaluateSanity();
} catch (Exception e) {
- logger.error("testStateCheck: After 10 sec sleep - evaluateSanity exception: ", e);
+ logger.error("testStateCheck: After 15 sec sleep - evaluateSanity exception: ", e);
sanityPass = false;
}
assertFalse(sanityPass); // expect sanity test to fail
- // undo dependency groups, jmx test properties settings and failed counter threshold
- myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
- myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
- myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, Integer.toString(IntegrityMonitorProperties.DEFAULT_FAILED_COUNTER_THRESHOLD));
- myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_MONITOR_INTERVAL));
- IntegrityMonitor.updateProperties(myProp);
-
- et = em.getTransaction();
-
- et.begin();
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
logger.debug("\n\ntestStateCheck: Exit\n\n");
}
public void testGetAllForwardProgressEntity() throws Exception{
logger.debug("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
-
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
// parameters are passed via a properties file
myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
- IntegrityMonitor.updateProperties(myProp);
-
- et = em.getTransaction();
- et.begin();
-
- // Make sure we start with the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
+ myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+ // Disable the integrity monitor so it will not interfere
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable dependency checking so it does not interfere
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+ // Disable the state audit
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable writing the FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
- IntegrityMonitor.deleteInstance();
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
- // Add a resources to put an entry in the forward progress table
+ // Add resource entries in the forward progress table
ForwardProgressEntity fpe = new ForwardProgressEntity();
ForwardProgressEntity fpe2 = new ForwardProgressEntity();
ForwardProgressEntity fpe3 = new ForwardProgressEntity();
@@ -881,42 +887,39 @@ public class IntegrityMonitorTest {
assertTrue(fpeList.size()==4);
- et = em.getTransaction();
-
- et.begin();
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
logger.debug("\nIntegrityMonitorTest: Exit testGetAllForwardProgressEntity\n\n");
}
public void testStateAudit() throws Exception{
logger.debug("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+ cleanDb();
+ cleanMyProp();
+ IntegrityMonitor.deleteInstance();
// parameters are passed via a properties file
- myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
- myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "100");
- IntegrityMonitor.updateProperties(myProp);
-
- et = em.getTransaction();
- et.begin();
- // Make sure we start with the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
+ // No Dependency Groups
+ myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+ // Don't use JMX
+ myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+ // Disable the internal sanity monitoring.
+ myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+ // Disable the dependency monitoring.
+ myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+ // Disable the refresh state audit
+ myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+ // Disable the test transaction
+ myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+ // Disable the write FPC
+ myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+ // Disable the State Audit we will call it directly
+ myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+ // Max interval for use in deciding if a FPC entry is stale in seconds
+ myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "120");
- IntegrityMonitor.deleteInstance();
IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+ StateManagement sm = im.getStateManager();
logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
// Add resources to put an entry in the forward progress table
@@ -1007,9 +1010,12 @@ public class IntegrityMonitorTest {
}
logger.debug("\n\n");
- logger.debug("IntegrityMonitorTest:testStateAudit: sleeping 2 sec");
- Thread.sleep(3000);
- logger.debug("IntegrityMonitorTest:testStateAudit: Awake!");
+ em.refresh(sme1);
+ assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+
+ logger.debug("IntegrityMonitorTest:testStateAudit: calling stateAudit()");
+ im.executeStateAudit();
+ logger.debug("IntegrityMonitorTest:testStateAudit: call to stateAudit() complete");
logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
smeList = query.getResultList();
@@ -1036,176 +1042,6 @@ public class IntegrityMonitorTest {
em.refresh(sme1);
assertTrue(sme1.getOpState().equals(StateManagement.DISABLED));
-
- //Now lock this IM
- StateManagement sm = im.getStateManager();
- sm.lock();
-
- //Give it time to write the db
- Thread.sleep(2000);
-
- //Put things back to their starting condition
- et = em.getTransaction();
- et.begin();
- sme1.setOpState(StateManagement.ENABLED);
- sme1.setAvailStatus(StateManagement.NULL_VALUE);
- em.persist(sme1);
- et.commit();
-
- //Now it should not update sme1
- logger.debug("IntegrityMonitorTest:testStateAudit: 2nd sleeping 2 sec");
- Thread.sleep(2000);
- logger.debug("IntegrityMonitorTest:testStateAudit: 2nd Awake!");
-
- logger.debug("\nIntegrityMonitorTest:testStateAudit 2nd getting list of StateManagementEntity entries\n\n");
- smeList = query.getResultList();
-
- logger.debug("\n\n");
- logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity 2nd entries");
- for(Object mySme : smeList){
- StateManagementEntity tmpSme = (StateManagementEntity) mySme;
- em.refresh(tmpSme);
- logger.debug("\n 2nd ResourceName: {}" +
- "\n AdminState: {}" +
- "\n OpState: {}" +
- "\n AvailStatus: {}" +
- "\n StandbyStatus: {}",
- tmpSme.getResourceName(),
- tmpSme.getAdminState(),
- tmpSme.getOpState(),
- tmpSme.getAvailStatus(),
- tmpSme.getStandbyStatus()
- );
- }
- logger.debug("\n\n");
-
- em.refresh(sme1);
- assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
-
- //Now create a reason for this IM to be disabled. Add a bogus dependency
- myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "Bogus_Node");
- IntegrityMonitor.updateProperties(myProp);
-
- //Restart the IM
- IntegrityMonitor.deleteInstance();
- im = IntegrityMonitor.getInstance(resourceName, myProp);
-
- //Give it time to initialize and check dependencies
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart) sleeping 10 sec");
- Thread.sleep(7000);
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart) Awake!");
-
- //Now unlock this IM. Now it should be unlocked, but disabled due to dependency
- sm.unlock();
-
- //Now check its state
- logger.debug("\nIntegrityMonitorTest:testStateAudit (restart) getting list of StateManagementEntity entries\n\n");
- smeList = query.getResultList();
-
- logger.debug("\n\n");
- logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart) entries");
- for(Object mySme : smeList){
- StateManagementEntity tmpSme = (StateManagementEntity) mySme;
- em.refresh(tmpSme);
-
- logger.debug("\n (restart) ResourceName: {}" +
- "\n AdminState: {}" +
- "\n OpState: {}" +
- "\n AvailStatus: {}" +
- "\n StandbyStatus: {}",
- tmpSme.getResourceName(),
- tmpSme.getAdminState(),
- tmpSme.getOpState(),
- tmpSme.getAvailStatus(),
- tmpSme.getStandbyStatus()
- );
-
- }
- logger.debug("\n\n");
-
- em.refresh(sme1);
- assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
-
- //Now lock this IM so it will not audit when it comes back up
- sm.lock();
-
- //Remove the bogus dependency and restart it
- myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
- IntegrityMonitor.updateProperties(myProp);
-
- //Restart the IM
- IntegrityMonitor.deleteInstance();
- im = IntegrityMonitor.getInstance(resourceName, myProp);
-
- //Give it time to initialize and check dependencies
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) sleeping 10 sec");
- Thread.sleep(7000);
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) Awake!");
-
- //Now check its state
- logger.debug("\nIntegrityMonitorTest:testStateAudit (restart2) getting list of StateManagementEntity entries\n\n");
- smeList = query.getResultList();
-
- logger.debug("\n\n");
- logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart2) entries");
- for(Object mySme : smeList){
- StateManagementEntity tmpSme = (StateManagementEntity) mySme;
- em.refresh(tmpSme);
-
- logger.debug("\n (restart2) ResourceName: {}" +
- "\n AdminState: {}" +
- "\n OpState: {}" +
- "\n AvailStatus: {}" +
- "\n StandbyStatus: {}",
- tmpSme.getResourceName(),
- tmpSme.getAdminState(),
- tmpSme.getOpState(),
- tmpSme.getAvailStatus(),
- tmpSme.getStandbyStatus()
- );
- }
- logger.debug("\n\n");
-
- em.refresh(sme1);
- assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
-
- //Make this IM coldstandby
- sm.demote();
- //Give it time to write the DB
- Thread.sleep(2000);
- //unlock it
- sm.unlock();
- //Give it time to write the DB
- Thread.sleep(2000);
-
- //Now check its state
- logger.debug("\nIntegrityMonitorTest:testStateAudit (restart3) getting list of StateManagementEntity entries\n\n");
- smeList = query.getResultList();
-
- logger.debug("\n\n");
- logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart3) entries");
- for(Object mySme : smeList){
- StateManagementEntity tmpSme = (StateManagementEntity) mySme;
- em.refresh(tmpSme);
-
- logger.debug("\n (restart3) ResourceName: {}" +
- "\n AdminState: {}" +
- "\n OpState: {}" +
- "\n AvailStatus: {}" +
- "\n StandbyStatus: {}",
- tmpSme.getResourceName(),
- tmpSme.getAdminState(),
- tmpSme.getOpState(),
- tmpSme.getAvailStatus(),
- tmpSme.getStandbyStatus()
- );
- }
- logger.debug("\n\n");
-
- //sme1 should not be changed because this IM is hotstandby and cannot change its state
- em.refresh(sme1);
- assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
-
//Now let's add sme2 to the mix
updateQuery = em.createQuery("UPDATE ForwardProgressEntity f "
+ "SET f.lastUpdated = :newDate "
@@ -1218,12 +1054,12 @@ public class IntegrityMonitorTest {
updateQuery.executeUpdate();
et.commit();
- //Finally, we want to promote this IM so it will disable sme1
- sm.promote();
//Give it a chance to write the DB and run the audit
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) sleeping 2 sec");
- Thread.sleep(3000);
- logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) Awake!");
+ logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) Running State Audit");
+ Thread.sleep(2000);
+ im.executeStateAudit();
+ Thread.sleep(1000);
+ logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) State Audit complete");
//Now check its state
logger.debug("\nIntegrityMonitorTest:testStateAudit (restart4) getting list of StateManagementEntity entries\n\n");
@@ -1255,16 +1091,6 @@ public class IntegrityMonitorTest {
em.refresh(sme2);
assertTrue(sme2.getOpState().equals(StateManagement.DISABLED));
- et = em.getTransaction();
- et.begin();
- // Make sure we leave the DB clean
- em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
- em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
- em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
-
- em.flush();
- et.commit();
-
logger.debug("\nIntegrityMonitorTest: Exit testStateAudit\n\n");
System.out.println("\n\ntestStateAudit: Exit\n\n");
}
diff --git a/integrity-monitor/src/test/resources/log4j.properties b/integrity-monitor/src/test/resources/log4j.properties
deleted file mode 100644
index efc18e9b..00000000
--- a/integrity-monitor/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,54 +0,0 @@
-###
-# ============LICENSE_START=======================================================
-# Integrity Monitor
-# ================================================================================
-# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END=========================================================
-###
-
-#
-# Use this properties for debugging and development.
-#
-#
-# For debug output, set root logger level to DEBUG and output to FILE and CONSOLE
-#log4j.rootLogger=DEBUG, FILE, CONSOLE
-log4j.rootLogger=INFO, FILE, CONSOLE
-
-# A1 is set to be a DailyRollingFileAppender.
-log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
-
-# Set the name of the file
-log4j.appender.FILE.File=IntegrityMonitor.log
-
-# Set the immediate flush to true (default)
-log4j.appender.FILE.ImmediateFlush=true
-
-# Set the threshold to debug mode
-log4j.appender.FILE.Threshold=debug
-
-# Set the append to false, should not overwrite
-log4j.appender.FILE.Append=true
-
-# Set the DatePattern
-log4j.appender.FILE.DatePattern='.'yyyy-MM-dd
-
-# A1 uses PatternLayout.
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{yyyy_MM_dd_HH_mm_ss_SSS} [%t] %-5p %l- %m%n
-
-# for Developments and Debugging
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d{yyyy_MM_dd_HH_mm_ss_SSS} [%t] %-5p %l- %m%n
diff --git a/integrity-monitor/src/test/resources/logback-test.xml b/integrity-monitor/src/test/resources/logback-test.xml
new file mode 100644
index 00000000..57a9d1c9
--- /dev/null
+++ b/integrity-monitor/src/test/resources/logback-test.xml
@@ -0,0 +1,47 @@
+<!--
+ ============LICENSE_START=======================================================
+ integrity-monitor
+ ================================================================================
+ Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ ================================================================================
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ ============LICENSE_END=========================================================
+ -->
+
+<!-- Controls the output of logs for JUnit tests -->
+
+<configuration>
+
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <Pattern>
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M\(%line\) - %msg%n
+ </Pattern>
+ </encoder>
+ </appender>
+ <appender name="FILE" class="ch.qos.logback.core.FileAppender">
+ <file>logs/debug.log</file>
+ <encoder>
+ <Pattern>
+ %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36}.%M\(%line\) - %msg%n
+ </Pattern>
+ </encoder>
+ </appender>
+
+ <root level="debug">
+ <appender-ref ref="STDOUT" />
+ <appender-ref ref="FILE" />
+ </root>
+
+</configuration>
+
diff --git a/integrity-monitor/src/test/resources/logback.xml b/integrity-monitor/src/test/resources/logback.xml
deleted file mode 100644
index 544162f3..00000000
--- a/integrity-monitor/src/test/resources/logback.xml
+++ /dev/null
@@ -1,209 +0,0 @@
-<!--
- ============LICENSE_START=======================================================
- Integrity Monitor
- ================================================================================
- Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
- ================================================================================
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ============LICENSE_END=========================================================
- -->
-
-<!-- Controls the output of logs for JUnit tests -->
-
-<configuration scan="true" scanPeriod="3 seconds" debug="true">
- <!--<jmxConfigurator /> -->
- <!-- directory path for all other type logs -->
- <property name="logDir" value="testingLogs" />
-
- <!-- directory path for debugging type logs -->
- <property name="debugDir" value="testingLogs" />
-
- <!-- specify the component name
- <ONAP-component-name>::= "MSO" | "DCAE" | "ASDC " | "AAI" |"Policy" | "SDNC" | "AC" -->
- <property name="componentName" value="common-modules"></property>
- <property name="subComponentName" value="integrity-monitor"></property>
-
- <!-- log file names -->
- <property name="errorLogName" value="error" />
- <property name="metricsLogName" value="metrics" />
- <property name="auditLogName" value="audit" />
- <property name="debugLogName" value="debug" />
-
- <property name="defaultPattern" value="%d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}|%X{requestId}|%X{serviceInstanceId}|%t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{clientIpAddress}|%c||%msg%n" />
- <!-- <property name="defaultPattern" value="%d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}|%X{RequestId}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ServerFQDN}|%X{RemoteHost}||%X{Timer}|%msg%n" /> -->
- <property name="logDirectory" value="${logDir}/${componentName}/${subComponentName}" />
- <property name="debugLogDirectory" value="${debugDir}/${componentName}/${subComponentName}" />
- <!--
- <property name="logDirectory" value="${logDir}/${componentName}/${subComponentName}" />
- <property name="debugLogDirectory" value="${debugDir}/${componentName}/${subComponentName}" />
- -->
- <!-- example from old log4j.properties: ${catalina.base}/logs/pdp-rest.log -->
- <!-- Example evaluator filter applied against console appender -->
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>${defaultPattern}</pattern>
- </encoder>
- </appender>
-
- <!-- ============================================================================ -->
- <!-- EELF Appenders -->
- <!-- ============================================================================ -->
-
- <!-- The EELFAppender is used to record events to the general application
- log -->
-
-
-
-
- <!-- EELF Audit Appender. This appender is used to record audit engine
- related logging events. The audit logger and appender are specializations
- of the EELF application root logger and appender. This can be used to segregate
- Policy engine events from other components, or it can be eliminated to record
- these events as part of the application root log. -->
-
- <appender name="EELFAudit"
- class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${auditLogName}.log</file>
- <rollingPolicy
- class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${auditLogName}.%i.log.zip
- </fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
- <triggeringPolicy
- class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
- </triggeringPolicy>
- <encoder>
- <pattern>${defaultPattern}</pattern>
- </encoder>
- </appender>
- <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>256</queueSize>
- <appender-ref ref="EELFAudit" />
- </appender>
-
-<appender name="EELFMetrics"
- class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${metricsLogName}.log</file>
- <rollingPolicy
- class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${metricsLogName}.%i.log.zip
- </fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
- <triggeringPolicy
- class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
- </triggeringPolicy>
- <encoder>
- <!-- <pattern>"%d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} -
- %msg%n"</pattern> -->
- <pattern>${defaultPattern}</pattern>
- </encoder>
- </appender>
-
-
- <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>256</queueSize>
- <appender-ref ref="EELFMetrics"/>
- </appender>
-
- <appender name="EELFError"
- class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${errorLogName}.log</file>
- <rollingPolicy
- class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${errorLogName}.%i.log.zip
- </fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>ERROR</level>
- </filter>
- <triggeringPolicy
- class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
- </triggeringPolicy>
- <encoder>
- <pattern>${defaultPattern}</pattern>
- </encoder>
- </appender>
-
- <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>256</queueSize>
- <appender-ref ref="EELFError"/>
- </appender>
-
- <appender name="EELFDebug"
- class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${debugLogDirectory}/${debugLogName}.log</file>
- <rollingPolicy
- class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${debugLogDirectory}/${debugLogName}.%i.log.zip
- </fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <!-- <level>INFO</level> -->
- <level>DEBUG</level>
- </filter>
- <triggeringPolicy
- class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
- </triggeringPolicy>
- <encoder>
- <pattern>${defaultPattern}</pattern>
- </encoder>
- </appender>
-
- <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender">
- <queueSize>256</queueSize>
- <appender-ref ref="EELFDebug" />
- <includeCallerData>true</includeCallerData>
- </appender>
-
-
- <!-- ============================================================================ -->
- <!-- EELF loggers -->
- <!-- ============================================================================ -->
-
- <logger name="com.att.eelf.audit" level="info" additivity="false">
- <appender-ref ref="asyncEELFAudit" />
- </logger>
-
- <logger name="com.att.eelf.metrics" level="info" additivity="false">
- <appender-ref ref="asyncEELFMetrics" />
- </logger>
-
- <logger name="com.att.eelf.error" level="error" additivity="false">
- <appender-ref ref="asyncEELFError" />
- </logger>
-
- <!-- <logger name="com.att.eelf.debug" level="info" additivity="false"> -->
- <logger name="com.att.eelf.debug" level="debug" additivity="false">
- <appender-ref ref="asyncEELFDebug" />
- </logger>
-
-
- <!-- <root level="INFO"> -->
- <root level="DEBUG">
- <appender-ref ref="asyncEELFDebug" />
- <appender-ref ref="asyncEELFError" />
- </root>
-
-</configuration>