Batch submit

[ECOMPD2TD-000] Update version.properties for iter 41

[US866186 1707] Adding methods to IntegrityMonitor to support state
audit.

[US866186 1710] Made more progress on teh stateAudit method of
IntegritryMontor.  Completed all except the addition of properties.

[US866186 1710] Addition of
IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS and logic to
stateAudit to determine whether or not to run

[US866186 1710] Created a JUnit for getAllForwardProgressEntity()
method and debugged and fixed it.

[US866186 1710] Making progress on the testStateAudit() JUnit.  Was able
to create forward progress entries and manually change the lastUpdated
date on an entry using a persistence update query.

[US866186 1710] First cut of stateAudit JUnit test.  Still need to test
more cases, but I got it working so far.

[US866186 1710] Completed the JUnit for stateAudit.  Added  5 new
scenarios.

Conflicts:

	version.properties

Change-Id: I7b5622a27021cf01b5d916b75824d8e51b5252eb
Signed-off-by: Ralph Straubs <rs8887@att.com>

2 files changed, 177 insertions, 1 deletions

diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
index 293bd11c..a4218942 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
@@ -21,6 +21,7 @@
 package org.openecomp.policy.common.im;
 
 import java.net.InetAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -101,6 +102,14 @@ public class IntegrityMonitor {
 	// This allows time for dependents to come up.
 	private long lastDependencyCheckTime = System.currentTimeMillis();
 	
+	// Time of the last state audit.  It is initialized at the time of the IM construction
+	private Date lastStateAuditTime = new Date();
+	
+	//Interval between state audits in ms.  We leave it turned off by default so that it will only
+	//be run on the nodes which we want doing the audit. In particular, we only want it to run
+	//on the droolspdps
+	private static long stateAuditIntervalMs = 0L;
+	
 	// the number of cycles since 'fpCounter' was last changed
 	private int missedCycles = 0;
 	
@@ -647,6 +656,53 @@ public class IntegrityMonitor {
 		return error_msg;
 	}
 	
+	public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity(){
+		if(logger.isDebugEnabled()){
+			logger.debug("getAllForwardProgressEntity: entry");
+		}
+		ArrayList<ForwardProgressEntity> fpList = new ArrayList<ForwardProgressEntity>();
+		// Start a transaction
+		EntityTransaction et = em.getTransaction();
+		et.begin();
+		try {
+			Query fquery = em.createQuery("Select e from ForwardProgressEntity e");
+			@SuppressWarnings("rawtypes")
+			List myList = fquery.setLockMode(
+					  LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+			synchronized(IMFLUSHLOCK){
+				et.commit();
+			}
+			if(logger.isDebugEnabled()){
+				logger.debug("getAllForwardProgressEntity: myList.size(): " + myList.size());
+			}
+			if(!myList.isEmpty()){
+				for(int i = 0; i < myList.size(); i++){
+					if(logger.isDebugEnabled()){
+						logger.debug("getAllForwardProgressEntity: myList.get(" + i +").getResourceName()" 
+							+ ": " + ((ForwardProgressEntity)myList.get(i)).getResourceName());
+					}
+					fpList.add((ForwardProgressEntity) myList.get(i));
+				}
+			}
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.commit();
+				}
+			}
+		} catch (Exception e) {
+			// log an error and continue
+			String msg = "getAllForwardProgessEntity DB read failed with exception: " + e;
+			logger.error(msg);
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
+			}
+		}
+		return fpList;
+	}
+	  
+	
 	private String jmxCheck(String dep) {
 		logger.debug("checking health of dependent by calling test() via JMX on resource: " + dep);
 
@@ -1117,6 +1173,14 @@ public class IntegrityMonitor {
 			}
 		}
 		
+		if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null){
+			try{
+				stateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
+			}catch(NumberFormatException e){
+				logger.warn("Ignored invalid property: " + IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS);
+			}
+		}
+		
 		
 		return;
 	}
@@ -1204,6 +1268,111 @@ public class IntegrityMonitor {
 	}
 	
 	/**
+	 * Look for "Forward Progress" on other nodes.  If they are not making forward progress,
+	 * check their operational state.  If it is not disabled, then disable them.
+	 */
+	public void stateAudit() {
+		
+		//TODO add stateAuditIntervalMs to the IntegrityMonitor properties here and in droolspdp
+		// monitoring interval checks
+		if (stateAuditIntervalMs <= 0) {
+			return; // stateAudit is disabled
+		}
+		
+		//Only run from nodes that are operational
+		if(stateManager.getOpState().equals(StateManagement.DISABLED)){
+			return;
+		}
+		if(stateManager.getAdminState().equals(StateManagement.LOCKED)){
+			return;
+		}
+		if(!stateManager.getStandbyStatus().equals(StateManagement.NULL_VALUE) &&
+				stateManager.getStandbyStatus()!= null){
+			if(!stateManager.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE)){
+				return;
+			}
+		}
+
+		Date date = new Date();		
+		long timeSinceLastStateAudit = date.getTime() - lastStateAuditTime.getTime(); 
+		if (timeSinceLastStateAudit < stateAuditIntervalMs){
+			return;
+		}
+
+		// Get all entries in the forwardprogressentity table
+		ArrayList<ForwardProgressEntity> fpList = getAllForwardProgressEntity();
+
+		// Check if each forwardprogressentity entry is current
+		for(ForwardProgressEntity fpe : fpList){
+			//If the this is my ForwardProgressEntity, continue
+			if(fpe.getResourceName().equals(IntegrityMonitor.resourceName)){
+				continue;
+			}
+			long diffMs = date.getTime() - fpe.getLastUpdated().getTime();
+			logger.debug("IntegrityMonitor.stateAudit(): diffMs = " + diffMs);
+
+			//Threshold for a stale entry
+			long staleMs = failedCounterThreshold * monitorInterval * 1000;
+			logger.debug("IntegrityMonitor.stateAudit(): staleMs = " + staleMs);
+
+			if(diffMs > staleMs){
+				//ForwardProgress is stale.  Disable it
+				// Start a transaction
+				EntityTransaction et = em.getTransaction();
+				et.begin();
+				StateManagementEntity sme = null;
+				try {
+					// query if StateManagement entry exists for fpe resource
+					Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+					query.setParameter("resource", fpe.getResourceName());
+
+					@SuppressWarnings("rawtypes")
+					List smList = query.setLockMode(
+							LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+					if (!smList.isEmpty()) {
+						// exists
+						sme = (StateManagementEntity) smList.get(0);
+						// refresh the object from DB in case cached data was returned
+						em.refresh(sme);
+						logger.debug("IntegrityMonitor.stateAudit(): Found entry in StateManagementEntity table for Resource=" + sme.getResourceName());
+					} else {
+						String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
+						logger.debug(msg);
+						logger.error(msg);
+					}
+					synchronized(IMFLUSHLOCK){
+						et.commit();
+					}
+				} catch (Exception e) {
+					// log an error
+					String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": StateManagementEntity DB read failed with exception: " + e;
+					logger.debug(msg);
+					logger.error(msg);
+					synchronized(IMFLUSHLOCK){
+						if(et.isActive()){
+							et.rollback();
+						}
+					}
+				}
+
+				if(sme != null){
+					if(!sme.getOpState().equals(StateManagement.DISABLED)){
+						logger.debug("IntegrityMonitor.stateAudit(): Changing OpStat = disabled for " + sme.getResourceName());
+						try {
+							stateManager.disableFailed(sme.getResourceName());
+						} catch (Exception e) {
+							String msg = "IntegrityMonitor.stateAudit(): Failed to disable " + sme.getResourceName();
+							logger.debug(msg);
+							logger.error(msg);
+						}
+					}
+				}
+			}// end if(diffMs > staleMs)
+		}// end for(ForwardProgressEntity fpe : fpList)
+		lastStateAuditTime = date;
+	}// end stateAudit()
+
+	/**
 	 * Execute a test transaction when test transaction interval has elapsed.
 	 */
 	private void checkTestTransaction() {
@@ -1319,7 +1488,7 @@ public class IntegrityMonitor {
 	
 	/**
 	 * The following nested class periodically performs the forward progress check,
-	 * checks dependencies and does a refresh state audit.
+	 * checks dependencies, does a refresh state audit and runs the stateAudit.
 	 */
 	class FPManager extends Thread {
 		
@@ -1372,6 +1541,12 @@ public class IntegrityMonitor {
 					// check if it is time to run the refreshStateAudit
 					IntegrityMonitor.this.refreshStateAudit();
 					
+					if(logger.isDebugEnabled()){
+						logger.debug("FPManager calling stateAudit()");
+					}
+					// check if it is time to run the stateAudit
+					IntegrityMonitor.this.stateAudit();
+					
 				} catch (Exception e) {
 					logger.debug("Ignore FPManager thread processing timer(s) exception: " + e);
 				}
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
index 8c3d7e6b..9b9ae550 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
@@ -51,5 +51,6 @@ public class IntegrityMonitorProperties {
 	public static final String TEST_VIA_JMX = "test_via_jmx";
 	public static final String JMX_FQDN = "jmx_fqdn";
 	public static final String MAX_FPC_UPDATE_INTERVAL = "max_fpc_update_interval";
+	public static final String STATE_AUDIT_INTERVAL_MS = "state_audit_interval_ms";
 	
 }