Batch submit

[ECOMPD2TD-000] Update version.properties for iter 41

[US866186 1707] Adding methods to IntegrityMonitor to support state
audit.

[US866186 1710] Made more progress on teh stateAudit method of
IntegritryMontor.  Completed all except the addition of properties.

[US866186 1710] Addition of
IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS and logic to
stateAudit to determine whether or not to run

[US866186 1710] Created a JUnit for getAllForwardProgressEntity()
method and debugged and fixed it.

[US866186 1710] Making progress on the testStateAudit() JUnit.  Was able
to create forward progress entries and manually change the lastUpdated
date on an entry using a persistence update query.

[US866186 1710] First cut of stateAudit JUnit test.  Still need to test
more cases, but I got it working so far.

[US866186 1710] Completed the JUnit for stateAudit.  Added  5 new
scenarios.

Conflicts:

	version.properties

Change-Id: I7b5622a27021cf01b5d916b75824d8e51b5252eb
Signed-off-by: Ralph Straubs <rs8887@att.com>

3 files changed, 580 insertions, 6 deletions

diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
index 293bd11c..a4218942 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitor.java
@@ -21,6 +21,7 @@
 package org.openecomp.policy.common.im;
 
 import java.net.InetAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -101,6 +102,14 @@ public class IntegrityMonitor {
 	// This allows time for dependents to come up.
 	private long lastDependencyCheckTime = System.currentTimeMillis();
 	
+	// Time of the last state audit.  It is initialized at the time of the IM construction
+	private Date lastStateAuditTime = new Date();
+	
+	//Interval between state audits in ms.  We leave it turned off by default so that it will only
+	//be run on the nodes which we want doing the audit. In particular, we only want it to run
+	//on the droolspdps
+	private static long stateAuditIntervalMs = 0L;
+	
 	// the number of cycles since 'fpCounter' was last changed
 	private int missedCycles = 0;
 	
@@ -647,6 +656,53 @@ public class IntegrityMonitor {
 		return error_msg;
 	}
 	
+	public ArrayList<ForwardProgressEntity> getAllForwardProgressEntity(){
+		if(logger.isDebugEnabled()){
+			logger.debug("getAllForwardProgressEntity: entry");
+		}
+		ArrayList<ForwardProgressEntity> fpList = new ArrayList<ForwardProgressEntity>();
+		// Start a transaction
+		EntityTransaction et = em.getTransaction();
+		et.begin();
+		try {
+			Query fquery = em.createQuery("Select e from ForwardProgressEntity e");
+			@SuppressWarnings("rawtypes")
+			List myList = fquery.setLockMode(
+					  LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+			synchronized(IMFLUSHLOCK){
+				et.commit();
+			}
+			if(logger.isDebugEnabled()){
+				logger.debug("getAllForwardProgressEntity: myList.size(): " + myList.size());
+			}
+			if(!myList.isEmpty()){
+				for(int i = 0; i < myList.size(); i++){
+					if(logger.isDebugEnabled()){
+						logger.debug("getAllForwardProgressEntity: myList.get(" + i +").getResourceName()" 
+							+ ": " + ((ForwardProgressEntity)myList.get(i)).getResourceName());
+					}
+					fpList.add((ForwardProgressEntity) myList.get(i));
+				}
+			}
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.commit();
+				}
+			}
+		} catch (Exception e) {
+			// log an error and continue
+			String msg = "getAllForwardProgessEntity DB read failed with exception: " + e;
+			logger.error(msg);
+			synchronized(IMFLUSHLOCK){
+				if(et.isActive()){
+					et.rollback();
+				}
+			}
+		}
+		return fpList;
+	}
+	  
+	
 	private String jmxCheck(String dep) {
 		logger.debug("checking health of dependent by calling test() via JMX on resource: " + dep);
 
@@ -1117,6 +1173,14 @@ public class IntegrityMonitor {
 			}
 		}
 		
+		if (prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS) != null){
+			try{
+				stateAuditIntervalMs = Long.parseLong(prop.getProperty(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS));
+			}catch(NumberFormatException e){
+				logger.warn("Ignored invalid property: " + IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS);
+			}
+		}
+		
 		
 		return;
 	}
@@ -1204,6 +1268,111 @@ public class IntegrityMonitor {
 	}
 	
 	/**
+	 * Look for "Forward Progress" on other nodes.  If they are not making forward progress,
+	 * check their operational state.  If it is not disabled, then disable them.
+	 */
+	public void stateAudit() {
+		
+		//TODO add stateAuditIntervalMs to the IntegrityMonitor properties here and in droolspdp
+		// monitoring interval checks
+		if (stateAuditIntervalMs <= 0) {
+			return; // stateAudit is disabled
+		}
+		
+		//Only run from nodes that are operational
+		if(stateManager.getOpState().equals(StateManagement.DISABLED)){
+			return;
+		}
+		if(stateManager.getAdminState().equals(StateManagement.LOCKED)){
+			return;
+		}
+		if(!stateManager.getStandbyStatus().equals(StateManagement.NULL_VALUE) &&
+				stateManager.getStandbyStatus()!= null){
+			if(!stateManager.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE)){
+				return;
+			}
+		}
+
+		Date date = new Date();		
+		long timeSinceLastStateAudit = date.getTime() - lastStateAuditTime.getTime(); 
+		if (timeSinceLastStateAudit < stateAuditIntervalMs){
+			return;
+		}
+
+		// Get all entries in the forwardprogressentity table
+		ArrayList<ForwardProgressEntity> fpList = getAllForwardProgressEntity();
+
+		// Check if each forwardprogressentity entry is current
+		for(ForwardProgressEntity fpe : fpList){
+			//If the this is my ForwardProgressEntity, continue
+			if(fpe.getResourceName().equals(IntegrityMonitor.resourceName)){
+				continue;
+			}
+			long diffMs = date.getTime() - fpe.getLastUpdated().getTime();
+			logger.debug("IntegrityMonitor.stateAudit(): diffMs = " + diffMs);
+
+			//Threshold for a stale entry
+			long staleMs = failedCounterThreshold * monitorInterval * 1000;
+			logger.debug("IntegrityMonitor.stateAudit(): staleMs = " + staleMs);
+
+			if(diffMs > staleMs){
+				//ForwardProgress is stale.  Disable it
+				// Start a transaction
+				EntityTransaction et = em.getTransaction();
+				et.begin();
+				StateManagementEntity sme = null;
+				try {
+					// query if StateManagement entry exists for fpe resource
+					Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+					query.setParameter("resource", fpe.getResourceName());
+
+					@SuppressWarnings("rawtypes")
+					List smList = query.setLockMode(
+							LockModeType.NONE).setFlushMode(FlushModeType.COMMIT).getResultList();
+					if (!smList.isEmpty()) {
+						// exists
+						sme = (StateManagementEntity) smList.get(0);
+						// refresh the object from DB in case cached data was returned
+						em.refresh(sme);
+						logger.debug("IntegrityMonitor.stateAudit(): Found entry in StateManagementEntity table for Resource=" + sme.getResourceName());
+					} else {
+						String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": resource not found in state management entity database table";
+						logger.debug(msg);
+						logger.error(msg);
+					}
+					synchronized(IMFLUSHLOCK){
+						et.commit();
+					}
+				} catch (Exception e) {
+					// log an error
+					String msg = "IntegrityMonitor.stateAudit(): " + fpe.getResourceName() + ": StateManagementEntity DB read failed with exception: " + e;
+					logger.debug(msg);
+					logger.error(msg);
+					synchronized(IMFLUSHLOCK){
+						if(et.isActive()){
+							et.rollback();
+						}
+					}
+				}
+
+				if(sme != null){
+					if(!sme.getOpState().equals(StateManagement.DISABLED)){
+						logger.debug("IntegrityMonitor.stateAudit(): Changing OpStat = disabled for " + sme.getResourceName());
+						try {
+							stateManager.disableFailed(sme.getResourceName());
+						} catch (Exception e) {
+							String msg = "IntegrityMonitor.stateAudit(): Failed to disable " + sme.getResourceName();
+							logger.debug(msg);
+							logger.error(msg);
+						}
+					}
+				}
+			}// end if(diffMs > staleMs)
+		}// end for(ForwardProgressEntity fpe : fpList)
+		lastStateAuditTime = date;
+	}// end stateAudit()
+
+	/**
 	 * Execute a test transaction when test transaction interval has elapsed.
 	 */
 	private void checkTestTransaction() {
@@ -1319,7 +1488,7 @@ public class IntegrityMonitor {
 	
 	/**
 	 * The following nested class periodically performs the forward progress check,
-	 * checks dependencies and does a refresh state audit.
+	 * checks dependencies, does a refresh state audit and runs the stateAudit.
 	 */
 	class FPManager extends Thread {
 		
@@ -1372,6 +1541,12 @@ public class IntegrityMonitor {
 					// check if it is time to run the refreshStateAudit
 					IntegrityMonitor.this.refreshStateAudit();
 					
+					if(logger.isDebugEnabled()){
+						logger.debug("FPManager calling stateAudit()");
+					}
+					// check if it is time to run the stateAudit
+					IntegrityMonitor.this.stateAudit();
+					
 				} catch (Exception e) {
 					logger.debug("Ignore FPManager thread processing timer(s) exception: " + e);
 				}
diff --git a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
index 8c3d7e6b..9b9ae550 100644
--- a/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
+++ b/integrity-monitor/src/main/java/org/openecomp/policy/common/im/IntegrityMonitorProperties.java
@@ -51,5 +51,6 @@ public class IntegrityMonitorProperties {
 	public static final String TEST_VIA_JMX = "test_via_jmx";
 	public static final String JMX_FQDN = "jmx_fqdn";
 	public static final String MAX_FPC_UPDATE_INTERVAL = "max_fpc_update_interval";
+	public static final String STATE_AUDIT_INTERVAL_MS = "state_audit_interval_ms";
 	
 }
diff --git a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
index 7375988e..acc9ad0e 100644
--- a/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
+++ b/integrity-monitor/src/test/java/org/openecomp/policy/common/im/test/IntegrityMonitorTest.java
@@ -22,6 +22,7 @@ package org.openecomp.policy.common.im.test;
 
 import static org.junit.Assert.*;
 
+import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.Properties;
@@ -31,6 +32,7 @@ import javax.persistence.EntityManagerFactory;
 import javax.persistence.EntityTransaction;
 import javax.persistence.Persistence;
 import javax.persistence.Query;
+import javax.persistence.TemporalType;
 
 import org.junit.After;
 import org.junit.AfterClass;
@@ -117,11 +119,13 @@ public class IntegrityMonitorTest {
 	@Ignore
 	@Test
 	public void runAllTests() throws Exception{
-		testSanityJmx();
-		testIM();
-		testSanityState();
-		testRefreshStateAudit();
-		testStateCheck();
+		//testSanityJmx();
+		//testIM();
+		//testSanityState();
+		//testRefreshStateAudit();
+		//testStateCheck();
+		//testGetAllForwardProgressEntity();
+		testStateAudit();
 	}
 
 	/*
@@ -833,4 +837,398 @@ public class IntegrityMonitorTest {
 		System.out.println("\n\ntestStateCheck: Exit\n\n");
 	}
 	
+	public void testGetAllForwardProgressEntity() throws Exception{
+		System.out.println("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
+		logger.debug("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
+		
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		IntegrityMonitor.updateProperties(myProp);
+		
+		et = em.getTransaction();
+		et.begin();
+
+		// Make sure we start with the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+
+		
+		IntegrityMonitor.deleteInstance();
+		IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+		
+		logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+		// Add a resources to put an entry in the forward progress table
+		ForwardProgressEntity fpe = new ForwardProgressEntity();
+		ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+		fpe.setFpcCount(0);
+		fpe.setResourceName("siteA_pap2");
+		fpe2.setFpcCount(0);
+		fpe2.setResourceName("siteB_pap1");
+		fpe3.setFpcCount(0);
+		fpe3.setResourceName("siteB_pap2");
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe);
+		em.persist(fpe2);
+		em.persist(fpe3);
+		em.flush();
+		et.commit();
+
+		logger.debug("\nIntegrityMonitorTest:testGetAllForwardProgressEntity Calling im.getAllForwardProgressEntity()\n\n");
+		ArrayList<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+		
+		assertTrue(fpeList.size()==4);
+		
+		et = em.getTransaction();
+		
+		et.begin();
+		// Make sure we leave the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+		
+		logger.debug("\nIntegrityMonitorTest: Exit testGetAllForwardProgressEntity\n\n");
+		System.out.println("\n\ntestGetAllForwardProgressEntity: Exit\n\n");
+	}
+	
+	public void testStateAudit() throws Exception{
+		System.out.println("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+		logger.debug("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+		
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "100");
+		IntegrityMonitor.updateProperties(myProp);
+		
+		et = em.getTransaction();
+		et.begin();
+
+		// Make sure we start with the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+
+		
+		IntegrityMonitor.deleteInstance();
+		IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp);
+		
+		logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+		// Add resources to put an entry in the forward progress table
+		Date staleDate = new Date(0);
+		ForwardProgressEntity fpe1 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+		fpe1.setFpcCount(0);
+		fpe1.setResourceName("siteA_pap2");
+		fpe2.setFpcCount(0);
+		fpe2.setResourceName("siteB_pap1");
+		fpe3.setFpcCount(0);
+		fpe3.setResourceName("siteB_pap2");
+		logger.debug("\nIntegrityMonitorTest: Creating StateManagementEntity entries\n\n");
+		StateManagementEntity sme1 = new StateManagementEntity();
+		StateManagementEntity sme2 = new StateManagementEntity();
+		StateManagementEntity sme3= new StateManagementEntity();
+		sme1.setResourceName("siteA_pap2");
+		sme1.setAdminState(StateManagement.UNLOCKED);
+		sme1.setOpState(StateManagement.ENABLED);
+		sme1.setAvailStatus(StateManagement.NULL_VALUE);
+		sme1.setStandbyStatus(StateManagement.NULL_VALUE);
+		sme2.setResourceName("siteB_pap1");
+		sme2.setAdminState(StateManagement.UNLOCKED);
+		sme2.setOpState(StateManagement.ENABLED);
+		sme2.setAvailStatus(StateManagement.NULL_VALUE);
+		sme2.setStandbyStatus(StateManagement.NULL_VALUE);
+		sme3.setResourceName("siteB_pap2");
+		sme3.setAdminState(StateManagement.UNLOCKED);
+		sme3.setOpState(StateManagement.ENABLED);
+		sme3.setAvailStatus(StateManagement.NULL_VALUE);
+		sme3.setStandbyStatus(StateManagement.NULL_VALUE);
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe1);
+		em.persist(fpe2);
+		em.persist(fpe3);
+		em.persist(sme1);
+		em.persist(sme2);
+		em.persist(sme3);
+		em.flush();
+		et.commit();
+		
+		Query updateQuery = em.createQuery("UPDATE ForwardProgressEntity f "
+				+ "SET f.lastUpdated = :newDate "
+				+ "WHERE f.resourceName=:resource");
+		updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+		updateQuery.setParameter("resource", fpe1.getResourceName());
+		
+		et = em.getTransaction();
+		et.begin();
+		updateQuery.executeUpdate();
+		et.commit();
+		
+		logger.debug("\nIntegrityMonitorTest:testStateAudit Calling im.getAllForwardProgressEntity()\n\n");
+		ArrayList<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:ForwardProgressEntity entries");
+		for(ForwardProgressEntity myFpe : fpeList){
+			logger.debug("\n    ResourceName: " + myFpe.getResourceName()
+					+ "\n        LastUpdated: " + myFpe.getLastUpdated());
+		}
+		logger.debug("\n\n");
+		
+		logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+		Query query = em.createQuery("SELECT s FROM StateManagementEntity s");
+		List smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		logger.debug("IntegrityMonitorTest:testStateAudit: sleeping 2 sec");
+		Thread.sleep(3000);
+		logger.debug("IntegrityMonitorTest:testStateAudit: Awake!");
+		
+		logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.DISABLED));
+		
+		
+		//Now lock this IM
+		StateManagement sm = im.getStateManager();
+		sm.lock();
+		
+		//Give it time to write the db
+		Thread.sleep(2000);
+
+		//Put things back to their starting condition		
+		et = em.getTransaction();
+		et.begin();
+		sme1.setOpState(StateManagement.ENABLED);
+		sme1.setAvailStatus(StateManagement.NULL_VALUE);
+		em.persist(sme1);
+		et.commit();
+
+		//Now it should not update sme1
+		logger.debug("IntegrityMonitorTest:testStateAudit: 2nd sleeping 2 sec");
+		Thread.sleep(2000);
+		logger.debug("IntegrityMonitorTest:testStateAudit: 2nd Awake!");
+		
+		logger.debug("\nIntegrityMonitorTest:testStateAudit 2nd getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity 2nd entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    2nd ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+		
+		//Now create a reason for this IM to be disabled.  Add a bogus dependency
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "Bogus_Node");
+		IntegrityMonitor.updateProperties(myProp);
+
+		//Restart the IM
+		IntegrityMonitor.deleteInstance();
+		im = IntegrityMonitor.getInstance(resourceName, myProp);
+		
+		//Give it time to initialize and check dependencies
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart) sleeping 10 sec");
+		Thread.sleep(7000);
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart) Awake!");
+		
+		//Now unlock this IM.  Now it should be unlocked, but disabled due to dependency
+		sm.unlock();
+		
+		//Now check its state
+		logger.debug("\nIntegrityMonitorTest:testStateAudit (restart) getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart) entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    (restart) ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+		
+		//Now lock this IM so it will not audit when it comes back up
+		sm.lock();
+		
+		//Remove the bogus dependency and restart it
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		IntegrityMonitor.updateProperties(myProp);
+
+		//Restart the IM
+		IntegrityMonitor.deleteInstance();
+		im = IntegrityMonitor.getInstance(resourceName, myProp);
+
+		//Give it time to initialize and check dependencies
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) sleeping 10 sec");
+		Thread.sleep(7000);
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart2) Awake!");
+		
+		//Now check its state
+		logger.debug("\nIntegrityMonitorTest:testStateAudit (restart2) getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart2) entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    (restart2) ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+		
+		//Make this IM coldstandby
+		sm.demote();
+		//Give it time to write the DB
+		Thread.sleep(2000);
+		//unlock it
+		sm.unlock();
+		//Give it time to write the DB
+		Thread.sleep(2000);
+		
+		//Now check its state
+		logger.debug("\nIntegrityMonitorTest:testStateAudit (restart3) getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart3) entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    (restart3) ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		//sme1 should not be changed because this IM is hotstandby and cannot change its state
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.ENABLED));
+		
+		//Now let's add sme2 to the mix
+		updateQuery = em.createQuery("UPDATE ForwardProgressEntity f "
+				+ "SET f.lastUpdated = :newDate "
+				+ "WHERE f.resourceName=:resource");
+		updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+		updateQuery.setParameter("resource", fpe2.getResourceName());
+		
+		et = em.getTransaction();
+		et.begin();
+		updateQuery.executeUpdate();
+		et.commit();
+		
+		//Finally, we want to promote this IM so it will disable sme1
+		sm.promote();
+		//Give it a chance to write the DB and run the audit
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) sleeping 2 sec");
+		Thread.sleep(3000);
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) Awake!");
+		
+		//Now check its state
+		logger.debug("\nIntegrityMonitorTest:testStateAudit (restart4) getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+		
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart4) entries");
+		for(Object mySme : smeList){
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug("\n    (restart4) ResourceName: " + tmpSme.getResourceName()
+					+ "\n        AdminState: " + tmpSme.getAdminState()
+					+ "\n        OpState: " + tmpSme.getOpState()
+					+ "\n        AvailStatus: " + tmpSme.getAvailStatus()
+					+ "\n        StandbyStatus: " + tmpSme.getStandbyStatus()
+					);
+		}
+		logger.debug("\n\n");
+		
+		em.refresh(sme1);
+		assertTrue(sme1.getOpState().equals(StateManagement.DISABLED));
+		
+		em.refresh(sme2);
+		assertTrue(sme2.getOpState().equals(StateManagement.DISABLED));
+		
+		et = em.getTransaction();
+		et.begin();
+		// Make sure we leave the DB clean
+		em.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		em.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		em.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+
+		em.flush();
+		et.commit();
+		
+		logger.debug("\nIntegrityMonitorTest: Exit testStateAudit\n\n");
+		System.out.println("\n\ntestStateAudit: Exit\n\n");
+	}
 }