Batch submit

[ECOMPD2TD-1073 1707] Removal of extra flush statements and addition
of rollbacks in catch blocks instead of commits.

[ECOMPD2TD-1073 1707] Adding some needed synchronized statements to
IntegrityMonitor.

[ECOMPD2TD-1159] - Loggers should be Serializable

By making loggers Serializable, they can be included in Drools
persistent data. 'EelfLogger' and 'SystemOutLogger' can do this
trivially, but 'Logger4J' needed some additional work, because it has a
non-serializable field 'log'.

[ECOMPD2TD-000] Fix versioning of org.openecomp.policy.* dependencies

[US866186 1707] First cut of stateCheck mod and non-working JUnit

[US866186 1707] Completed the coding for the task TA1998344 which adds
a check of forward progress for dependencies.

[US866186 1707] Completed updates to IntegrityMonitor.stateCheck and
IntegrityMonitorTest which includes addition of a JUnit for stateCheck
and control of the order of JUnit execution.

[US865296] ECOMP Policy Logging Compliance, add TargetEntity and
TargetServiceName, remove unit from ElapsedTime

[US865296] ECOMP Policy Logging Compliance, set audit log statuscode to
'COMPLETE' instead of N/A

[US865296] ECOMP Policy Logging Compliance, remove time unit (seconds
and milliseconds) from ElapsedTime for logging compliance

[US866186 1707] Cleaned up IntegrityAudit JUnit tests.

[US866186 1707] IntegrityMonitor JUnit clean up

[US866186 1707] Re-added missing classes that were erroneously deleted.

[US865296] - add get/setters and inits for required log fields

Change-Id: I76ef4606ed6832ed48eaca68e72839a05c8bc3a8
Signed-off-by: Ralph Straubs <rs8887@att.com>

6 files changed, 360 insertions, 160 deletions

diff --git a/integrity-monitor/config/policyLogger.properties b/integrity-monitor/config/policyLogger.properties
new file mode 100644
index 00000000..1e7187f2
--- /dev/null
+++ b/integrity-monitor/config/policyLogger.properties
@@ -0,0 +1,44 @@
+###
+# ============LICENSE_START=======================================================
+# Integrity Audit
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+
+################################### Set concurrentHashMap and timer info  #######################
+#Timer initial delay and the delay between in milliseconds before task is to be execute.
+timer.delay.time=1000
+#Timer scheduleAtFixedRate period - time in milliseconds between successive task executions.
+check.interval= 30000
+#Longest time an event info can be stored in the concurrentHashMap for logging - in seconds. 
+event.expired.time=86400
+#Size of the concurrentHashMap which stores the event starting time, etc - when its size reaches this limit, the Timer gets executed 
+#to remove all expired records from this concurrentHashMap.
+concurrentHashMap.limit=5000
+#Size of the concurrentHashMap - when its size drops to this point, stop the Timer
+stop.check.point=2500
+################################### Set logging format #############################################
+# set EELF for EELF logging format, set LOG4J for using log4j, set SYSTEMOUT for using system.out.println
+logger.type=EELF
+#################################### Set level for EELF or SYSTEMOUT logging ##################################
+# Set level for debug file. Set DEBUG to enable .info, .warn and .debug; set INFO for enable .info and .warn; set OFF to disable all 
+debugLogger.level=INFO
+# Set level for metrics file. Set OFF to disable; set ON to enable
+metricsLogger.level=ON
+# Set level for error file. Set OFF to disable; set ON to enable
+error.level=ON
+# Set level for audit file. Set OFF to disable; set ON to enable
+audit.level=ON
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
index 6c575ab7..293bd11c 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
@@ -37,14 +37,15 @@ import javax.persistence.LockModeType;
 import javax.persistence.Persistence;
 import javax.persistence.Query;
 
-//import org.apache.log4j.Logger;
-
-import org.openecomp.policy.common.im.jmx.*;
 import org.openecomp.policy.common.im.jpa.ForwardProgressEntity;
 import org.openecomp.policy.common.im.jpa.ResourceRegistrationEntity;
 import org.openecomp.policy.common.im.jpa.StateManagementEntity;
-import org.openecomp.policy.common.logging.flexlogger.FlexLogger; 
+import org.openecomp.policy.common.logging.flexlogger.FlexLogger;
 import org.openecomp.policy.common.logging.flexlogger.Logger;
+//import org.apache.log4j.Logger;
+import org.openecomp.policy.common.im.jmx.ComponentAdmin;
+import org.openecomp.policy.common.im.jmx.ComponentAdminMBean;
+import org.openecomp.policy.common.im.jmx.JmxAgentConnection;
 
 /**
  * IntegrityMonitor
@@ -309,7 +310,6 @@ public class IntegrityMonitor {
         	em.persist(rrx);
         	// flush to the DB
         	synchronized(IMFLUSHLOCK){
-        		em.flush();
         		et.commit();
         	}
         
@@ -442,59 +442,145 @@ public class IntegrityMonitor {
 		}
 
 	}
-	
-	private String stateCheck(String dep) {
+
+	/*
+	 * This method checks the forward progress counter and the state of
+	 * a dependency.  If the dependency is unavailable or failed, an
+	 * error message is created which is checked when evaluateSanity interface
+	 * is called.  If the error message is set then the  evaluateSanity
+	 * will return an error.  
+	 */
+	public String stateCheck(String dep) {
 		logger.debug("checking state of dependent resource: " + dep);
-		
-		// get state management entry for dependent resource
-		StateManagementEntity stateManagementEntity = null;
+
 		String error_msg = null;
-		try {
-			// Start a transaction
-			EntityTransaction et = em.getTransaction();
-			et.begin();
+		ForwardProgressEntity forwardProgressEntity = null;
+		StateManagementEntity stateManagementEntity = null;
+		
+		// Start a transaction
+		EntityTransaction et = em.getTransaction();
+		et.begin();
 
-			// query if StateManagement entry exists for dependent resource
-			Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+		try{
+			Query query = em.createQuery("Select p from ForwardProgressEntity p where p.resourceName=:resource");
 			query.setParameter("resource", dep);
 
 			@SuppressWarnings("rawtypes")
-			List smList = query.setLockMode(
-					  LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
-			if (!smList.isEmpty()) {
-				// exist 
-				stateManagementEntity = (StateManagementEntity) smList.get(0);
+			List fpList = query.setLockMode(
+					LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+
+			if (!fpList.isEmpty()) {
+				// exists
+				forwardProgressEntity = (ForwardProgressEntity) fpList.get(0);
 				// refresh the object from DB in case cached data was returned
-        		em.refresh(stateManagementEntity);
-				logger.debug("Found entry in StateManagementEntity table for dependent Resource=" + dep);
+				em.refresh(forwardProgressEntity);
+				logger.debug("Found entry in ForwardProgressEntity table for dependent Resource=" + dep);
 			} else {
-				error_msg = dep + ": resource not found in state management entity database table";
+				error_msg = dep + ": resource not found in ForwardProgressEntity database table";
+				logger.debug(error_msg);
 				logger.error(error_msg);
 			}
-
 			synchronized(IMFLUSHLOCK){
 				et.commit();
 			}
-		} catch (Exception e) {
+		}
+		catch(Exception ex){
 			// log an error
-			error_msg = dep + ": StateManagementEntity DB read failed with exception: " + e;
+			error_msg = dep + ": ForwardProgressEntity DB operation failed with exception: " + ex;
+			logger.debug(error_msg);
 			logger.error(error_msg);
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
+			}
+		}
+
+		if(error_msg==null){
+			// Start a transaction
+			et = em.getTransaction();
+			et.begin();
+			try {
+				// query if StateManagement entry exists for dependent resource
+				Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+				query.setParameter("resource", dep);
+
+				@SuppressWarnings("rawtypes")
+				List smList = query.setLockMode(
+						LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+				if (!smList.isEmpty()) {
+					// exist 
+					stateManagementEntity = (StateManagementEntity) smList.get(0);
+					// refresh the object from DB in case cached data was returned
+					em.refresh(stateManagementEntity);
+					logger.debug("Found entry in StateManagementEntity table for dependent Resource=" + dep);
+				} else {
+					error_msg = dep + ": resource not found in state management entity database table";
+					logger.debug(error_msg);
+					logger.error(error_msg);
+				}
+
+				synchronized(IMFLUSHLOCK){
+					et.commit();
+				}
+			} catch (Exception e) {
+				// log an error
+				error_msg = dep + ": StateManagementEntity DB read failed with exception: " + e;
+				logger.debug(error_msg);
+				logger.error(error_msg);
+				synchronized(IMFLUSHLOCK){
+					if(et.isActive()){
+						et.rollback();
+					}
+				}
+			}
+		}
+
+		//verify that the ForwardProgress is current (check last_updated)
+		if(error_msg==null){
+			Date date = new Date();
+			long diffMs = date.getTime() - forwardProgressEntity.getLastUpdated().getTime();
+			logger.debug("IntegrityMonitor.stateCheck(): diffMs = " + diffMs);
+
+			//Threshold for a stale entry
+			long staleMs = failedCounterThreshold * monitorInterval * 1000;
+			logger.debug("IntegrityMonitor.stateCheck(): staleMs = " + staleMs);
+
+			if(diffMs > staleMs){
+				//ForwardProgress is stale.  Disable it
+				try {
+					if(!stateManagementEntity.getOpState().equals(StateManagement.DISABLED)){
+						logger.debug("IntegrityMonitor.stateCheck(): Changing OpStat = disabled for " + dep);
+						stateManager.disableFailed(dep);
+					}
+				} catch (Exception e) {
+					String msg = "IntegrityMonitor.stateCheck(): Failed to diableFail dependent resource = " + dep 
+							+ "; " + e.getMessage();
+					logger.debug(msg);
+					logger.error(msg);
+				}
+			}
 		}
 		
 		// check operation, admin and standby states of dependent resource
 		if (error_msg == null) {
 			if ((stateManager.getAdminState() != null) && stateManagementEntity.getAdminState().equals(StateManagement.LOCKED)) {
 				error_msg = dep + ": resource is administratively locked";
+				logger.debug(error_msg);
 				logger.error(error_msg);
 			} else if ((stateManager.getOpState() != null) && stateManagementEntity.getOpState().equals(StateManagement.DISABLED)) {
 				error_msg = dep + ": resource is operationally disabled";
+				logger.debug(error_msg);
 				logger.error(error_msg);
 			} else if ((stateManager.getStandbyStatus() != null) && stateManagementEntity.getStandbyStatus().equals(StateManagement.COLD_STANDBY)) {
 				error_msg = dep + ": resource is cold standby";
+				logger.debug(error_msg);
 				logger.error(error_msg);
 			}
 		}
 		
+		String returnMsg = "IntegrityMonitor.stateCheck(): returned error_msg: " + error_msg;
+		logger.debug(returnMsg);
 		return error_msg;
 	}
 	
@@ -551,6 +637,11 @@ public class IntegrityMonitor {
 			// log an error and continue
 			error_msg = dep + ": ForwardProgressEntity DB read failed with exception: " + e;
 			logger.error(error_msg);
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
+			}
 		}
 		
 		return error_msg;
@@ -563,11 +654,10 @@ public class IntegrityMonitor {
 
 		// get the JMX URL from the database
 		String jmxUrl = null;
+		// Start a transaction
+		EntityTransaction et = em.getTransaction();
+		et.begin();
 		try {
-			// Start a transaction
-			EntityTransaction et = em.getTransaction();
-			et.begin();
-
 			// query if ResourceRegistration entry exists for resourceName
 			Query rquery = em.createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
 			rquery.setParameter("rn", dep);
@@ -595,6 +685,11 @@ public class IntegrityMonitor {
 		} catch (Exception e) {
 			error_msg = dep + ": ResourceRegistrationEntity DB read failed with exception: " + e;
 			logger.error(error_msg);
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
+			}
 		}
 
 
@@ -876,7 +971,6 @@ public class IntegrityMonitor {
 				em.persist(fpx);
 				// flush to the DB and commit
 				synchronized(IMFLUSHLOCK){
-					em.flush();
 					et.commit();
 				}
 			}
@@ -887,8 +981,10 @@ public class IntegrityMonitor {
 			}
         } catch (Exception e) {
         	try {
-        		if (et.isActive()) {
-        			et.rollback();
+        		synchronized(IMFLUSHLOCK){
+        			if (et.isActive()) {
+        				et.rollback();
+        			}
         		}
         	} catch (Exception e1) {
         		// ignore
@@ -965,18 +1061,6 @@ public class IntegrityMonitor {
 			}
 		}
 		
-		/***********************
-		// followers are a comma separated list of resource names
-		if (prop.getProperty(IntegrityMonitorProperties.SS_FOLLOWERS) != null) {
-			try {
-				followers = prop.getProperty(IntegrityMonitorProperties.SS_FOLLOWERS).split(",");
-				logger.debug("followers property = " + Arrays.toString(followers));
-			} catch (Exception e) {
-				logger.warn("Ignored invalid property: " + IntegrityMonitorProperties.SS_FOLLOWERS);
-			}
-		}
-		**************************/
-		
 		// dependency_groups are a semi-colon separated list of groups
 		// each group is a comma separated list of resource names
 		// For ex. dependency_groups = site_1.pap_1,site_1.pap_2 ; site_1.pdp_1, site_1.pdp_2
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/StateManagement.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/StateManagement.java
index 14d35e1d..fd0f3d9f 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/StateManagement.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/StateManagement.java
@@ -129,16 +129,11 @@ public class StateManagement extends Observable {
           logger.debug("Persist adminstrative state, resourceName = " + this.resourceName); 
           em.persist(sm);
           synchronized(FLUSHLOCK){
-        	  em.flush();
-        	  if(et.isActive()){
-        		  et.commit(); 
-        	  }
+       		  et.commit(); 
           }
         } else {
         	synchronized(FLUSHLOCK){
-        		if(et.isActive()){
-        			et.commit(); 
-        		}
+       			et.commit(); 
         	}
         }
         
@@ -147,13 +142,13 @@ public class StateManagement extends Observable {
 
         logger.debug("StateManagement: constructor end, resourceName: " + this.resourceName);
       } catch(Exception ex) {
+    	  logger.error("StateManagement: constructor caught unexpected exception: " + ex);
+    	  ex.printStackTrace();
     	  synchronized(FLUSHLOCK){
     		  if(et.isActive()){
-    			  et.commit();
+    			  et.rollback();
     		  }
     	  }
-    	  ex.printStackTrace();
-    	  logger.error("StateManagement: constructor caught unexpected exception: " + ex);
     	  throw new Exception("StateManagement: Exception: " + ex.toString());
       } 
   }
@@ -186,23 +181,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(ADMIN_STATE);
 
 			  logger.debug("StateManagement: initializeState() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.initializeState() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.initializeState() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.initializeState() Exception: " + ex);
 		  }
 	  }
@@ -237,23 +229,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(ADMIN_STATE);
 
 			  logger.debug("StateManagement: lock() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.lock() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.lock() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.lock() Exception: " + ex.toString());
 		  } 
 	  }
@@ -287,23 +276,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(ADMIN_STATE);
 
 			  logger.debug("StateManagement: unlock() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.unlock() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.unlock() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.unlock() Exception: " + ex);
 		  }
 	  }
@@ -338,23 +324,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(OPERATION_STATE);
 
 			  logger.debug("StateManagement enableNotFailed() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.enableNotFailed() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.enableNotFailed() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.enableNotFailed() Exception: " + ex);
 		  }
 	  }
@@ -387,23 +370,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(OPERATION_STATE);
 
 			  logger.debug("StateManagement: disableFailed() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.disableFailed() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.disableFailed() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.disableFailed() Exception: " + ex);
 		  }
 	  }
@@ -442,10 +422,7 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(OPERATION_STATE);
@@ -453,13 +430,13 @@ public class StateManagement extends Observable {
 			  logger.debug("StateManagement: disableFailed(otherResourceName) operation completed, resourceName = " 
 					  + otherResourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.disableFailed(otherResourceName) caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.disableFailed(otherResourceName) caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.disableFailed(otherResourceName) Exception: " + ex);
 		  }
 	  }
@@ -493,23 +470,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(OPERATION_STATE);
 
 			  logger.debug("StateManagement: disableDependency() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.disableDependency() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.disableDependency() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.disableDependency() Exception: " + ex);
 		  }
 	  }
@@ -544,23 +518,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush(); 
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(OPERATION_STATE);
 
 			  logger.debug("StateManagement: enableNoDependency() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.enableNoDependency() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.enableNoDependency() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.enableNoDependency() Exception: " + ex);
 		  }
 	  }
@@ -597,7 +568,6 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
 				  if(et.isActive()){
 					  et.commit(); 
 				  }
@@ -605,13 +575,13 @@ public class StateManagement extends Observable {
 			  setChanged();
 			  notifyObservers(STANDBY_STATUS);
 		  }catch(Exception ex){
+			  logger.error("StateManagement.promote() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.promote() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.promote() Exception: " + ex);
 		  }
 
@@ -651,23 +621,20 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  setChanged();
 			  notifyObservers(STANDBY_STATUS); 
 
 			  logger.debug("StateManagement: demote() operation completed, resourceName = " + this.resourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.demote() caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.demote() caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.demote() Exception: " + ex);
 		  }
 	  }
@@ -710,22 +677,19 @@ public class StateManagement extends Observable {
 
 			  em.persist(sm);
 			  synchronized(FLUSHLOCK){
-				  em.flush();
-				  if(et.isActive()){
-					  et.commit(); 
-				  }
+				  et.commit(); 
 			  }
 			  //We don't notify observers because this is assumed to be a remote resource
 
 			  logger.debug("StateManagement: demote(otherResourceName) operation completed, resourceName = " + otherResourceName);
 		  } catch(Exception ex) {
+			  logger.error("StateManagement.demote(otherResourceName) caught unexpected exception: " + ex);
+			  ex.printStackTrace();
 			  synchronized(FLUSHLOCK){
 				  if(et.isActive()){
-					  et.commit();
+					  et.rollback();
 				  }
 			  }
-			  ex.printStackTrace();
-			  logger.error("StateManagement.demote(otherResourceName) caught unexpected exception: " + ex);
 			  throw new Exception("StateManagement.demote(otherResourceName) Exception: " + ex);
 		  }
 	  }
@@ -914,6 +878,7 @@ public String getOpState()
 
 		String standbyStatus = null;
 		
+		// The transaction is required for the LockModeType
 		EntityTransaction et = em.getTransaction();
 		if(!et.isActive()){
 			et.begin();
@@ -940,17 +905,19 @@ public String getOpState()
 				logger.error("getStandbyStatus: resourceName =" + otherResourceName
 						+ " not found in statemanagemententity table");
 			}
+			synchronized(FLUSHLOCK){
+				et.commit();
+			}
 		} catch (Exception e) {
-			e.printStackTrace();
 			logger.error("getStandbyStatus: Caught Exception attempting to get statemanagemententity record, message='"
 					+ e.getMessage() + "'");
-		}
-		synchronized(FLUSHLOCK){
-			if(et.isActive()){
-				et.commit();
+			e.printStackTrace();
+			synchronized(FLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
 			}
 		}
-
 		if (logger.isDebugEnabled()) {
 			logger.debug("getStandbyStatus: Returning standbyStatus="
 					+ standbyStatus);
@@ -989,27 +956,21 @@ public String getOpState()
 					  + stateManagementEntity.getStandbyStatus());
 			  em.remove(stateManagementEntity);
 		  }
+		  synchronized(FLUSHLOCK){
+			  et.commit();
+		  }
 	  }catch(Exception ex){
+		  logger.error("StateManagement.deleteAllStateManagementEntities() caught Exception: " + ex);
+		  ex.printStackTrace();
 		  synchronized(FLUSHLOCK){
 			  if(et.isActive()){
-				  et.commit();
+				  et.rollback();
 			  }
 		  }
-		  ex.printStackTrace();
-		  logger.error("StateManagement.deleteAllStateManagementEntities() caught Exception: " + ex);
 	  }
-
-	  /*
-	   * End transaction.
-	   */
-	  synchronized(FLUSHLOCK){
-		  if(et.isActive()){
-			  et.commit();
-		  }
+	  if(logger.isDebugEnabled()){
+		  logger.info("deleteAllStateManagementEntities: Exiting");
 	  }
-
-	  logger.info("deleteAllStateManagementEntities: Exiting");
-
   }
 
 }
diff --git a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
index 68a6cf2f..7375988e 100644
--- a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
+++ b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
@@ -53,6 +53,11 @@ import org.openecomp.policy.common.im.jpa.StateManagementEntity;
 import org.openecomp.policy.common.logging.flexlogger.FlexLogger; 
 import org.openecomp.policy.common.logging.flexlogger.Logger;
 
+/*
+ * All JUnits are designed to run in the local development environment
+ * where they have write privileges and can execute time-sensitive
+ * tasks.
+ */
 public class IntegrityMonitorTest {
 	private static Logger logger = FlexLogger.getLogger(IntegrityMonitorTest.class);
 	private static Properties myProp;
@@ -103,6 +108,21 @@ public class IntegrityMonitorTest {
 		// clear jmx remote port setting
 		systemProps.remove("com.sun.management.jmxremote.port");
 	}
+	
+	/*
+	 * The following runs all tests and controls the order of execution. If you allow
+	 * the tests to execute individually, you cannot predict the order and some
+	 * conflicts occur.
+	 */
+	@Ignore
+	@Test
+	public void runAllTests() throws Exception{
+		testSanityJmx();
+		testIM();
+		testSanityState();
+		testRefreshStateAudit();
+		testStateCheck();
+	}
 
 	/*
 	 * The following test verifies the following test cases:
@@ -114,8 +134,6 @@ public class IntegrityMonitorTest {
 	 * Unlock
 	 * Unlock restart
 	 */
-	@Ignore // Test passed 10/18/16 
-	@Test
 	public void testSanityJmx() throws Exception {
 		System.out.println("\nIntegrityMonitorTest: Entering testSanityJmx\n\n");
 		
@@ -267,8 +285,6 @@ public class IntegrityMonitorTest {
 	}
 	
 
-	@Ignore  // Test passed 10/18/16 
-	@Test
 	public void testIM() throws Exception {
 		System.out.println("\nIntegrityMonitorTest: Entering testIM\n\n");
 		
@@ -554,8 +570,6 @@ public class IntegrityMonitorTest {
 	}
 	
 
-	@Ignore  // Test passed 10/18/16 
-	@Test
 	public void testSanityState() throws Exception {
 		System.out.println("\nIntegrityMonitorTest: Entering testSanityState\n\n");
 		
@@ -618,8 +632,6 @@ public class IntegrityMonitorTest {
 		System.out.println("\n\ntestSanityState: Exit\n\n");
 	}
 	
-	@Ignore  // Test passed 10/18/16 
-	@Test
 	public void testRefreshStateAudit() throws Exception {
 		logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Enter\n\n");
 
@@ -731,4 +743,94 @@ public class IntegrityMonitorTest {
 
 		logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Exit\n\n");
 	}
+	
+	public void testStateCheck() throws Exception {
+		System.out.println("\nIntegrityMonitorTest: Entering testStateCheck\n\n");
+		
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "group1_dep1");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, "1");
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "10");
+		IntegrityMonitor.updateProperties(myProp);
+		/*
+		 *  The default monitorInterval is 30 and the default failedCounterThreshold is 3
+		 *  Since stateCheck() uses the faileCounterThreshold * monitorInterval to determine
+		 *  if an entry is stale, it will be stale after 30 seconds.
+		 */
+		
+		et = em.getTransaction();
+		et.begin();
+
+		// Make sure we start with the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+
+		IntegrityMonitor.deleteInstance();
+		
+		IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+		
+		// Add a group1 dependent resources to put an entry in the forward progress table
+		// This sets lastUpdated to the current time
+		ForwardProgressEntity fpe = new ForwardProgressEntity();
+		fpe.setFpcCount(0);
+		fpe.setResourceName("group1_dep1");
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe);
+		em.flush();
+		et.commit();
+
+		//Now add new group1 stateManager instances
+		StateManagement sm2 = new StateManagement(emf, "group1_dep1");
+		
+		boolean sanityPass = true;
+		//Thread.sleep(15000);
+		Thread.sleep(5000);
+		try {
+			im.evaluateSanity();
+		} catch (Exception e) {
+			System.out.println("testStateCheck: After 15 sec sleep - evaluateSanity exception: " + e);
+			sanityPass = false;
+		}
+		assertTrue(sanityPass);  // expect sanity test to pass
+		
+		//now wait 30 seconds.  The dependency entry should now be stale and the sanitry check should fail
+		
+		sanityPass = true;
+		//Thread.sleep(30000);
+		Thread.sleep(10000);
+		try {
+			im.evaluateSanity();
+		} catch (Exception e) {
+			System.out.println("testStateCheck: After 10 sec sleep - evaluateSanity exception: " + e);
+			sanityPass = false;
+		}
+		assertFalse(sanityPass);  // expect sanity test to fail
+		
+		// undo dependency groups, jmx test properties settings and failed counter threshold
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, Integer.toString(IntegrityMonitorProperties.DEFAULT_FAILED_COUNTER_THRESHOLD));
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, Integer.toString(IntegrityMonitorProperties.DEFAULT_MONITOR_INTERVAL));
+		IntegrityMonitor.updateProperties(myProp);
+
+		et = em.getTransaction();
+		
+		et.begin();
+		// Make sure we leave the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+		
+		System.out.println("\n\ntestStateCheck: Exit\n\n");
+	}
+	
 }
diff --git a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateManagementTest.java b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateManagementTest.java
index 3247d781..f51f5ac7 100644
--- a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateManagementTest.java
+++ b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateManagementTest.java
@@ -41,6 +41,7 @@ import javax.persistence.Query;
 
 
 
+
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -49,7 +50,6 @@ import org.junit.Ignore;
 import org.junit.Test;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.openecomp.policy.common.im.StateManagement;
 import org.openecomp.policy.common.im.StateTransition;
 import org.openecomp.policy.common.im.StandbyStatusException; 
@@ -57,6 +57,11 @@ import org.openecomp.policy.common.im.StateChangeNotifier;
 import org.openecomp.policy.common.logging.flexlogger.FlexLogger; 
 import org.openecomp.policy.common.logging.flexlogger.Logger;
 
+/*
+ * All JUnits are designed to run in the local development environment
+ * where they have write privileges and can execute time-sensitive
+ * tasks.
+ */
 public class StateManagementTest {
 	private static Logger logger = FlexLogger.getLogger(StateManagementTest.class);
 	
diff --git a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateTransitionTest.java b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateTransitionTest.java
index b0e6e18e..d76cad0d 100644
--- a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateTransitionTest.java
+++ b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/StateTransitionTest.java
@@ -50,7 +50,11 @@ import org.openecomp.policy.common.im.StandbyStatusException;
 import org.openecomp.policy.common.im.StateChangeNotifier; 
 import org.openecomp.policy.common.logging.flexlogger.FlexLogger; 
 import org.openecomp.policy.common.logging.flexlogger.Logger;
-
+/*
+ * All JUnits are designed to run in the local development environment
+ * where they have write privileges and can execute time-sensitive
+ * tasks.
+ */
 public class StateTransitionTest {
 	private static Logger logger = FlexLogger.getLogger(StateTransitionTest.class);