summaryrefslogtreecommitdiffstats
path: root/integrity-monitor/src/main/java/org
diff options
context:
space:
mode:
Diffstat (limited to 'integrity-monitor/src/main/java/org')
-rw-r--r--integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java177
-rw-r--r--integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java1
2 files changed, 177 insertions, 1 deletions
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
index 293bd11c..a4218942 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
@@ -21,6 +21,7 @@
package org.openecomp.policy.common.im;
import java.net.InetAddress;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
@@ -101,6 +102,14 @@ public class IntegrityMonitor {
// This allows time for dependents to come up.
private long lastDependencyCheckTime = System.currentTimeMillis();
+ // Time of the last state audit. It is initialized at the time of the IM construction
+ private Date lastStateAuditTime = new Date();
+
+ //Interval between state audits in ms. We leave it turned off by default so that it will only
+ //be run on the nodes which we want doing the audit. In particular, we only want it to run
+ //on the droolspdps
+ private static long stateAuditIntervalMs = 0L;
+
// the number of cycles since 'fpCounter' was last changed
private int missedCycles = 0;
@@ -647,6 +656,53 @@ public class IntegrityMonitor {
return error_msg;
}
+ public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity(){
+ if(logger.isDebugEnabled()){
+ logger.debug("getAllForwardProgressEntity: entry");
+ }
+ ArrayList<ForwardProgressEntity> fpList = new ArrayList<ForwardProgressEntity>();
+ // Start a transaction
+ EntityTransaction et = em.getTransaction();
+ et.begin();
+ try {
+ Query fquery = em.createQuery("Select e from ForwardProgressEntity e");
+ @SuppressWarnings("rawtypes")
+ List myList = fquery.setLockMode(
+ LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ synchronized(IMFLUSHLOCK){
+ et.commit();
+ }
+ if(logger.isDebugEnabled()){
+ logger.debug("getAllForwardProgressEntity: myList.size(): " + myList.size());
+ }
+ if(!myList.isEmpty()){
+ for(int i = 0; i < myList.size(); i++){
+ if(logger.isDebugEnabled()){
+ logger.debug("getAllForwardProgressEntity: myList.get(" + i +").getResourceName()"
+ + ": " + ((ForwardProgressEntity)myList.get(i)).getResourceName());
+ }
+ fpList.add((ForwardProgressEntity) myList.get(i));
+ }
+ }
+ synchronized(IMFLUSHLOCK){
+ if(et.isActive()){
+ et.commit();
+ }
+ }
+ } catch (Exception e) {
+ // log an error and continue
+ String msg = "getAllForwardProgessEntity DB read failed with exception: " + e;
+ logger.error(msg);
+ synchronized(IMFLUSHLOCK){
+ if(et.isActive()){
+ et.rollback();
+ }
+ }
+ }
+ return fpList;
+ }
+
+
private String jmxCheck(String dep) {
logger.debug("checking health of dependent by calling test() via JMX on resource: " + dep);
@@ -1117,6 +1173,14 @@ public class IntegrityMonitor {
}
}
+ if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null){
+ try{
+ stateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
+ }catch(NumberFormatException e){
+ logger.warn("Ignored invalid property: " + IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS);
+ }
+ }
+
return;
}
@@ -1204,6 +1268,111 @@ public class IntegrityMonitor {
}
/**
+ * Look for "Forward Progress" on other nodes. If they are not making forward progress,
+ * check their operational state. If it is not disabled, then disable them.
+ */
+ public void stateAudit() {
+
+ //TODO add stateAuditIntervalMs to the IntegrityMonitor properties here and in droolspdp
+ // monitoring interval checks
+ if (stateAuditIntervalMs <= 0) {
+ return; // stateAudit is disabled
+ }
+
+ //Only run from nodes that are operational
+ if(stateManager.getOpState().equals(StateManagement.DISABLED)){
+ return;
+ }
+ if(stateManager.getAdminState().equals(StateManagement.LOCKED)){
+ return;
+ }
+ if(!stateManager.getStandbyStatus().equals(StateManagement.NULL_VALUE) &&
+ stateManager.getStandbyStatus()!= null){
+ if(!stateManager.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE)){
+ return;
+ }
+ }
+
+ Date date = new Date();
+ long timeSinceLastStateAudit = date.getTime() - lastStateAuditTime.getTime();
+ if (timeSinceLastStateAudit < stateAuditIntervalMs){
+ return;
+ }
+
+ // Get all entries in the forwardprogressentity table
+ ArrayList<ForwardProgressEntity> fpList = getAllForwardProgressEntity();
+
+ // Check if each forwardprogressentity entry is current
+ for(ForwardProgressEntity fpe : fpList){
+ //If the this is my ForwardProgressEntity, continue
+ if(fpe.getResourceName().equals(IntegrityMonitor.resourceName)){
+ continue;
+ }
+ long diffMs = date.getTime() - fpe.getLastUpdated().getTime();
+ logger.debug("IntegrityMonitor.stateAudit(): diffMs = " + diffMs);
+
+ //Threshold for a stale entry
+ long staleMs = failedCounterThreshold * monitorInterval * 1000;
+ logger.debug("IntegrityMonitor.stateAudit(): staleMs = " + staleMs);
+
+ if(diffMs > staleMs){
+ //ForwardProgress is stale. Disable it
+ // Start a transaction
+ EntityTransaction et = em.getTransaction();
+ et.begin();
+ StateManagementEntity sme = null;
+ try {
+ // query if StateManagement entry exists for fpe resource
+ Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+ query.setParameter("resource", fpe.getResourceName());
+
+ @SuppressWarnings("rawtypes")
+ List smList = query.setLockMode(
+ LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+ if (!smList.isEmpty()) {
+ // exists
+ sme = (StateManagementEntity) smList.get(0);
+ // refresh the object from DB in case cached data was returned
+ em.refresh(sme);
+ logger.debug("IntegrityMonitor.stateAudit(): Found entry in StateManagementEntity table for Resource=" + sme.getResourceName());
+ } else {
+ String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
+ logger.debug(msg);
+ logger.error(msg);
+ }
+ synchronized(IMFLUSHLOCK){
+ et.commit();
+ }
+ } catch (Exception e) {
+ // log an error
+ String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": StateManagementEntity DB read failed with exception: " + e;
+ logger.debug(msg);
+ logger.error(msg);
+ synchronized(IMFLUSHLOCK){
+ if(et.isActive()){
+ et.rollback();
+ }
+ }
+ }
+
+ if(sme != null){
+ if(!sme.getOpState().equals(StateManagement.DISABLED)){
+ logger.debug("IntegrityMonitor.stateAudit(): Changing OpStat = disabled for " + sme.getResourceName());
+ try {
+ stateManager.disableFailed(sme.getResourceName());
+ } catch (Exception e) {
+ String msg = "IntegrityMonitor.stateAudit(): Failed to disable " + sme.getResourceName();
+ logger.debug(msg);
+ logger.error(msg);
+ }
+ }
+ }
+ }// end if(diffMs > staleMs)
+ }// end for(ForwardProgressEntity fpe : fpList)
+ lastStateAuditTime = date;
+ }// end stateAudit()
+
+ /**
* Execute a test transaction when test transaction interval has elapsed.
*/
private void checkTestTransaction() {
@@ -1319,7 +1488,7 @@ public class IntegrityMonitor {
/**
* The following nested class periodically performs the forward progress check,
- * checks dependencies and does a refresh state audit.
+ * checks dependencies, does a refresh state audit and runs the stateAudit.
*/
class FPManager extends Thread {
@@ -1372,6 +1541,12 @@ public class IntegrityMonitor {
// check if it is time to run the refreshStateAudit
IntegrityMonitor.this.refreshStateAudit();
+ if(logger.isDebugEnabled()){
+ logger.debug("FPManager calling stateAudit()");
+ }
+ // check if it is time to run the stateAudit
+ IntegrityMonitor.this.stateAudit();
+
} catch (Exception e) {
logger.debug("Ignore FPManager thread processing timer(s) exception: " + e);
}
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
index 8c3d7e6b..9b9ae550 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
@@ -51,5 +51,6 @@ public class IntegrityMonitorProperties {
public static final String TEST_VIA_JMX = "test_via_jmx";
public static final String JMX_FQDN = "jmx_fqdn";
public static final String MAX_FPC_UPDATE_INTERVAL = "max_fpc_update_interval";
+ public static final String STATE_AUDIT_INTERVAL_MS = "state_audit_interval_ms";
}