Add test coverage to integrity-monitor

Removed "test" from package name.
Refactored tests, creating common IntegrityMonitorTestBase.
Turned log/print statements into assertEquals in StateTransitionTest.
Turned log/print statements into assertEquals in StateManagementTest.
Modified AllSeemsWellTest to use IntegrityMonitorTestBase.
Modified IntegrityMonitorTest to use IntegrityMonitorTestBase.
Added several hooks to IntegrityMonitor to control timers at a
finer granularity.
Added hooks to IntegrityMonitor to control the FPManager thread.
Remove hooks for refresh timer, as property suffices.
Added assertException() and assertNoException() methods to
IntegrityMonitorTestBase, and then replaced most "try/catch" blocks
with calls to those methods.
Updated StateManagement to close EntityManagers.
Modify pom to remove scope from utils dependency.
Fix some comments and remove an unneeded EntityTransaction variable.

Change-Id: Ic0789d26f985a40a35f618343fa4e88aa473b2b3
Issue-ID: POLICY-582
Signed-off-by: Jim Hahn <jrh3@att.com>

1 files changed, 907 insertions, 0 deletions

diff --git a/integrity-monitor/src/test/java/org/onap/policy/common/im/IntegrityMonitorTest.java b/integrity-monitor/src/test/java/org/onap/policy/common/im/IntegrityMonitorTest.java
new file mode 100644
index 00000000..3704b07d
--- /dev/null
+++ b/integrity-monitor/src/test/java/org/onap/policy/common/im/IntegrityMonitorTest.java
@@ -0,0 +1,907 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Integrity Monitor
+ * ================================================================================
+ * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.common.im;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import javax.persistence.EntityTransaction;
+import javax.persistence.Query;
+import javax.persistence.TemporalType;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.onap.policy.common.im.jpa.ForwardProgressEntity;
+import org.onap.policy.common.im.jpa.ResourceRegistrationEntity;
+import org.onap.policy.common.im.jpa.StateManagementEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * All JUnits are designed to run in the local development environment
+ * where they have write privileges and can execute time-sensitive
+ * tasks.
+ */
+public class IntegrityMonitorTest extends IntegrityMonitorTestBase {
+	private static Logger logger = LoggerFactory.getLogger(IntegrityMonitorTest.class);
+
+	private static Properties myProp;
+	private static EntityTransaction et;
+	private static String resourceName;
+
+	private BlockingQueue<CountDownLatch> queue;
+
+	@BeforeClass
+	public static void setUpClass() throws Exception {
+		IntegrityMonitorTestBase.setUpBeforeClass(DEFAULT_DB_URL_PREFIX + IntegrityMonitorTest.class.getSimpleName());
+
+		resourceName = IntegrityMonitorTestBase.siteName + "." + IntegrityMonitorTestBase.nodeType;
+	}
+
+	@AfterClass
+	public static void tearDownClass() throws Exception {
+		IntegrityMonitorTestBase.tearDownAfterClass();
+	}
+
+	@Before
+	public void setUp() throws Exception {
+		super.setUpTest();
+
+		myProp = makeProperties();
+		et = null;
+	}
+
+	@After
+	public void tearDown() throws Exception {
+		if (et != null && et.isActive()) {
+			try {
+				et.rollback();
+
+			} catch (RuntimeException e) {
+				logger.error("cannot rollback transaction", e);
+			}
+		}
+
+		super.tearDownTest();
+	}
+
+	/*
+	 * The following test verifies the following test cases: New Install New
+	 * Install - Bad Dependency data Recovery from bad dependency data Lock Lock
+	 * restart Unlock Unlock restart
+	 */
+	@Test
+	public void testSanityJmx() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testSanityJmx\n\n");
+
+		String dependent = "group1_logparser";
+
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, dependent);
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "true");
+		// Disable the integrity monitor so it will not interfere
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the state audit
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable the write FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+		// Speed up the check
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "1");
+		// Fail dependencies after three seconds
+		myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "3");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+		logger.debug(
+				"\n\ntestSanityJmx starting im state \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				im.getStateManager().getAdminState(), im.getStateManager().getOpState(),
+				im.getStateManager().getAvailStatus(), im.getStateManager().getStandbyStatus());
+		// add an entry to Resource registration table in the DB for the
+		// dependent resource
+
+		et = em.getTransaction();
+		et.begin();
+		Query rquery = em.createQuery("Select r from ResourceRegistrationEntity r where r.resourceName=:rn");
+		rquery.setParameter("rn", dependent);
+
+		@SuppressWarnings("rawtypes")
+		List rrList = rquery.getResultList();
+		ResourceRegistrationEntity rrx = null;
+		if (rrList.isEmpty()) {
+			// register resource by adding entry to table in DB
+			logger.debug("Adding resource {}  to ResourceRegistration table", dependent);
+			rrx = new ResourceRegistrationEntity();
+			// set columns in entry
+			rrx.setResourceName(dependent);
+			rrx.setResourceUrl("service:jmx:somewhere:9999");
+			rrx.setNodeType("logparser");
+			rrx.setSite("siteA");
+		}
+		em.persist(rrx);
+		// flush to the DB
+		em.flush();
+
+		// commit transaction
+		et.commit();
+
+		// wait for the FPManager to check dependency health
+		waitStep();
+
+		assertException(im, imx -> {
+			imx.evaluateSanity();
+		});
+
+		// undo dependency groups and jmx test properties settings
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		IntegrityMonitor.updateProperties(myProp);
+
+		logger.debug("\ntestSantityJmx ending properties: {}", myProp);
+
+		// We know at this point that the IM is disable-dependency. We want to
+		// be
+		// sure it will recover from this condition since the properties were
+		// updated.
+
+		logger.debug(
+				"\n\ntestSanityJmx ending im state\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				im.getStateManager().getAdminState(), im.getStateManager().getOpState(),
+				im.getStateManager().getAvailStatus(), im.getStateManager().getStandbyStatus());
+
+		logger.debug("\ntestSanityJmx restarting the IntegrityMonitor");
+		// Create a new instance. It should recover from the disabled-dependency
+		// condition
+		im = makeMonitor(resourceName, myProp);
+
+		logger.debug(
+				"\n\ntestSanityJmx state after creating new im\n"
+						+ "AdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				im.getStateManager().getAdminState(), im.getStateManager().getOpState(),
+				im.getStateManager().getAvailStatus(), im.getStateManager().getStandbyStatus());
+
+		// Verify the state
+		assertEquals(StateManagement.UNLOCKED, im.getStateManager().getAdminState());
+		assertEquals(StateManagement.ENABLED, im.getStateManager().getOpState());
+		assertEquals(StateManagement.NULL_VALUE, im.getStateManager().getAvailStatus());
+		assertEquals(StateManagement.NULL_VALUE, im.getStateManager().getStandbyStatus());
+
+		// Test state manager via the IntegrityMonitor
+		StateManagement sm = im.getStateManager();
+
+		// Verify lock state
+		sm.lock();
+		logger.debug("\n\nsm.lock()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.LOCKED, sm.getAdminState());
+
+		// Verify lock persists across a restart
+		logger.debug("\ntestSanityJmx restarting the IntegrityMonitor");
+		// Create a new instance. It should come up with the admin state locked
+		im = makeMonitor(resourceName, myProp);
+		sm = im.getStateManager();
+		logger.debug(
+				"\n\ntestSanityJmx restart with AdminState=locked"
+						+ "\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.LOCKED, sm.getAdminState());
+
+		// Verify unlock
+		sm.unlock();
+		logger.debug(
+				"\n\ntestSanityJmx sm.unlock\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.UNLOCKED, sm.getAdminState());
+
+		// Verify unlock restart
+		logger.debug("\ntestSanityJmx restarting the IntegrityMonitor");
+		// Create a new instance. It should come up with the admin state locked
+		im = makeMonitor(resourceName, myProp);
+		sm = im.getStateManager();
+		logger.debug(
+				"\n\ntestSanityJmx restart with AdminState=unlocked\n"
+						+ "AdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.UNLOCKED, sm.getAdminState());
+
+		logger.debug("\n\ntestSanityJmx: Exit\n\n");
+	}
+
+	@Test
+	public void testIM() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testIM\n\n");
+
+		// Disable the integrity monitor so it will not interfere
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable dependency checking
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the state audit
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable writing the FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+
+		logger.debug("\n\nim initial state: \nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				im.getStateManager().getAdminState(), im.getStateManager().getOpState(),
+				im.getStateManager().getAvailStatus(), im.getStateManager().getStandbyStatus());
+
+		waitStep();
+
+		// test evaluate sanity
+		assertNoException(im, imx -> {
+			imx.evaluateSanity();
+		});
+
+		// Test startTransaction - should work since it is unlocked
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		// Test state manager via the IntegrityMonitor
+		StateManagement sm = im.getStateManager();
+
+		sm.lock();
+
+		logger.debug("\n\nsm.lock()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.LOCKED, sm.getAdminState());
+
+		// test startTransaction. It should fail since it is locked
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.unlock();
+		logger.debug("\n\nsm.unlock()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.UNLOCKED, sm.getAdminState());
+
+		// test startTransaction. It should succeed
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.disableDependency();
+		logger.debug(
+				"\n\nsm.disableDependency()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.DISABLED, sm.getOpState());
+		assertEquals(StateManagement.DEPENDENCY, sm.getAvailStatus());
+
+		// test startTransaction. It should succeed since standby status is null
+		// and unlocked
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.enableNoDependency();
+
+		logger.debug(
+				"\n\nsm.enableNoDependency()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.ENABLED, sm.getOpState());
+		// test startTransaction. It should succeed since standby status is null
+		// and unlocked
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.disableFailed();
+		logger.debug("\n\nsm.disableFailed()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.DISABLED, sm.getOpState());
+		assertEquals(StateManagement.FAILED, sm.getAvailStatus());
+		// test startTransaction. It should succeed since standby status is null
+		// and unlocked
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.enableNotFailed();
+
+		logger.debug(
+				"\n\nsm.enabledNotFailed()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.ENABLED, sm.getOpState());
+		// test startTransaction. It should succeed since standby status is null
+		// and unlocked
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.demote();
+
+		logger.debug("\n\nsm.demote()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.HOT_STANDBY, sm.getStandbyStatus());
+
+		// test startTransaction. It should fail since it is standby
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.promote();
+
+		logger.debug("\n\nsm.promote()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.PROVIDING_SERVICE, sm.getStandbyStatus());
+
+		// test startTransaction. It should succeed since it is providing
+		// service
+		assertNoException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		// Test the multi-valued availability status
+		sm.disableDependency();
+		sm.disableFailed();
+
+		logger.debug(
+				"\n\nsm.disableDependency(), sm.disableFailed\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.DEPENDENCY_FAILED, sm.getAvailStatus());
+
+		// Test startTransaction. Should fail since standby status is cold
+		// standby
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.enableNoDependency();
+
+		logger.debug(
+				"\n\nsm.enableNoDependency()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.FAILED, sm.getAvailStatus());
+		// Test startTransaction. Should fail since standby status is cold
+		// standby
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.disableDependency();
+		sm.enableNotFailed();
+
+		logger.debug(
+				"\n\nsm.disableDependency(),sm.enableNotFailed()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+
+		assertEquals(StateManagement.DEPENDENCY, sm.getAvailStatus());
+		// Test startTransaction. Should fail since standby status is cold
+		// standby
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		sm.enableNoDependency();
+		logger.debug(
+				"\n\nsm.enableNoDependency()\nAdminState = {}\nOpState() = {}\nAvailStatus = {}\nStandbyStatus = {}\n",
+				sm.getAdminState(), sm.getOpState(), sm.getAvailStatus(), sm.getStandbyStatus());
+		assertEquals(StateManagement.ENABLED, sm.getOpState());
+		// test startTransaction. It should fail since standby status is hot
+		// standby
+		assertException(im, imx -> {
+			imx.startTransaction();
+		});
+
+		logger.debug("\n\ntestIM: Exit\n\n");
+	}
+
+	@Test
+	public void testSanityState() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testSanityState\n\n");
+
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "group1_dep1,group1_dep2; group2_dep1");
+		// Disable the integrity monitor so it will not interfere
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable dependency checking so it does not interfere
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+		// Disable the state audit
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable writing the FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+		// Max interval for use in deciding if a FPC entry is stale in seconds
+		myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "120");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+
+		waitStep();
+
+		// Add a group1 dependent resources to put an entry in the forward
+		// progress table
+		ForwardProgressEntity fpe = new ForwardProgressEntity();
+		ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+		fpe.setFpcCount(0);
+		fpe.setResourceName("group1_dep1");
+		fpe2.setFpcCount(0);
+		fpe2.setResourceName("group1_dep2");
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe);
+		em.persist(fpe2);
+		em.flush();
+		et.commit();
+
+		// Add a group2 dependent resource to the StateManagementEntity DB table
+		// and set its admin state to locked
+		// Expect sanity test to fail.
+		StateManagement stateManager = new StateManagement(emf, "group2_dep1");
+		stateManager.lock();
+
+		new StateManagement(emf, "group1_dep1");
+		new StateManagement(emf, "group1_dep2");
+
+		// Call the dependency check directly instead of waiting for FPManager
+		// to do it.
+		logger.debug("\n\nIntegrityMonitor.testSanityState: calling im.dependencyCheck()\n\n");
+		im.dependencyCheck();
+		assertException(im, imx -> {
+			imx.evaluateSanity();
+		});
+
+		logger.debug("\n\ntestSanityState: Exit\n\n");
+	}
+
+	@Test
+	public void testRefreshStateAudit() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Enter\n\n");
+
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		// Disable the integrity monitor so it will not interfere
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable dependency checking so it does not interfere
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+		// Disable the state audit
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable writing the FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+
+		waitStep();
+
+		// the state here is unlocked, enabled, null, null
+		StateManagementEntity sme = null;
+
+		Query query = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+
+		query.setParameter("resource", resourceName);
+
+		// Just test that we are retrieving the right object
+		@SuppressWarnings("rawtypes")
+		List resourceList = query.getResultList();
+		if (!resourceList.isEmpty()) {
+			// exist
+			sme = (StateManagementEntity) resourceList.get(0);
+			em.refresh(sme);
+
+			logger.debug(
+					"??? -- Retrieve StateManagementEntity from database --\nsme.getResourceName() = {}\n"
+							+ "sme.getAdminState() = {}\nsme.getOpState() = {}\nsme.getAvailStatus() = {}\nsme.getStandbyStatus() = {}",
+					sme.getResourceName(), sme.getAdminState(), sme.getOpState(), sme.getAvailStatus(),
+					sme.getStandbyStatus());
+
+			assertEquals(StateManagement.UNLOCKED, sme.getAdminState());
+			assertEquals(StateManagement.ENABLED, sme.getOpState());
+			assertEquals(StateManagement.NULL_VALUE, sme.getAvailStatus());
+			assertEquals(StateManagement.NULL_VALUE, sme.getStandbyStatus());
+			logger.debug("--");
+		} else {
+			logger.debug("Record not found, resourceName: " + resourceName);
+			fail("missing record");
+		}
+
+		et = em.getTransaction();
+		et.begin();
+
+		sme.setStandbyStatus(StateManagement.COLD_STANDBY);
+		em.persist(sme);
+		em.flush();
+		et.commit();
+
+		// Run the refreshStateAudit
+		im.executeRefreshStateAudit();
+
+		// The refreshStateAudit should run and change the state to
+		// unlocked,enabled,null,hotstandby
+		StateManagementEntity sme1 = null;
+
+		Query query1 = em.createQuery("Select p from StateManagementEntity p where p.resourceName=:resource");
+
+		query1.setParameter("resource", resourceName);
+
+		@SuppressWarnings("rawtypes")
+		List resourceList1 = query1.getResultList();
+		if (!resourceList1.isEmpty()) {
+			// exist
+			sme1 = (StateManagementEntity) resourceList1.get(0);
+			em.refresh(sme1);
+			logger.debug(
+					"??? -- Retrieve StateManagementEntity from database --\nsme1.getResourceName() = {}\n"
+							+ "sme1.getAdminState() = {}\nsme1.getOpState() = {}\nsme1.getAvailStatus() = {}\nsme1.getStandbyStatus() = {}",
+					sme1.getResourceName(), sme1.getAdminState(), sme1.getOpState(), sme1.getAvailStatus(),
+					sme1.getStandbyStatus());
+
+			assertEquals(StateManagement.UNLOCKED, sme1.getAdminState());
+			assertEquals(StateManagement.ENABLED, sme1.getOpState());
+			assertEquals(StateManagement.NULL_VALUE, sme1.getAvailStatus());
+			assertEquals(StateManagement.HOT_STANDBY, sme1.getStandbyStatus());
+			logger.debug("--");
+		} else {
+			logger.debug("Record not found, resourceName: " + resourceName);
+			fail("record not found");
+		}
+
+		logger.debug("\nIntegrityMonitorTest: testRefreshStateAudit Exit\n\n");
+	}
+
+	@Test
+	public void testStateCheck() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testStateCheck\n\n");
+
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "group1_dep1");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		myProp.put(IntegrityMonitorProperties.FAILED_COUNTER_THRESHOLD, "1");
+		/*
+		 * The monitorInterval is set to 10 and the failedCounterThreshold is 1
+		 * because stateCheck() uses the faileCounterThreshold * monitorInterval
+		 * to determine if an entry is stale, it will be stale after 10 seconds.
+		 */
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "5");
+		/*
+		 * We accelerate the test transaction and write FPC intervals because we
+		 * don't want there to be any chance of a FPC failure because of the
+		 * short monitor interval
+		 */
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "1");
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "2");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// The maximum time in seconds to determine that a FPC entry is stale
+		myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "5");
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "5");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+
+		// Note: do ***NOT*** do waitStep() here
+
+		// Add a group1 dependent resources to put an entry in the forward
+		// progress table
+		// This sets lastUpdated to the current time
+		ForwardProgressEntity fpe = new ForwardProgressEntity();
+		fpe.setFpcCount(0);
+		fpe.setResourceName("group1_dep1");
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe);
+		em.flush();
+		et.commit();
+
+		new StateManagement(emf, "group1_dep1");
+
+		assertNoException(im, imx -> {
+			imx.evaluateSanity();
+		});
+
+		// wait for FPManager to perform dependency health check. Once that's
+		// done,
+		// it should now be stale and the sanity check should fail
+		waitStep();
+
+		assertException(im, imx -> {
+			imx.evaluateSanity();
+		});
+
+		logger.debug("\n\ntestStateCheck: Exit\n\n");
+	}
+
+	@Test
+	public void testGetAllForwardProgressEntity() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testGetAllForwardProgressEntity\n\n");
+		// parameters are passed via a properties file
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		// Disable the integrity monitor so it will not interfere
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable dependency checking so it does not interfere
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+		// Disable the state audit
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable writing the FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+		waitStep();
+
+		logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+		// Add resource entries in the forward progress table
+		ForwardProgressEntity fpe = new ForwardProgressEntity();
+		ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+		fpe.setFpcCount(0);
+		fpe.setResourceName("siteA_pap2");
+		fpe2.setFpcCount(0);
+		fpe2.setResourceName("siteB_pap1");
+		fpe3.setFpcCount(0);
+		fpe3.setResourceName("siteB_pap2");
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe);
+		em.persist(fpe2);
+		em.persist(fpe3);
+		em.flush();
+		et.commit();
+
+		logger.debug(
+				"\nIntegrityMonitorTest:testGetAllForwardProgressEntity Calling im.getAllForwardProgressEntity()\n\n");
+		List<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+
+		assertEquals(4, fpeList.size());
+
+		logger.debug("\nIntegrityMonitorTest: Exit testGetAllForwardProgressEntity\n\n");
+	}
+
+	@Test
+	public void testStateAudit() throws Exception {
+		logger.debug("\nIntegrityMonitorTest: Entering testStateAudit\n\n");
+
+		// parameters are passed via a properties file
+
+		// No Dependency Groups
+		myProp.put(IntegrityMonitorProperties.DEPENDENCY_GROUPS, "");
+		// Don't use JMX
+		myProp.put(IntegrityMonitorProperties.TEST_VIA_JMX, "false");
+		// Disable the internal sanity monitoring.
+		myProp.put(IntegrityMonitorProperties.FP_MONITOR_INTERVAL, "-1");
+		// Disable the dependency monitoring.
+		myProp.put(IntegrityMonitorProperties.CHECK_DEPENDENCY_INTERVAL, "-1");
+		// Disable the refresh state audit
+		myProp.put(IntegrityMonitorProperties.REFRESH_STATE_AUDIT_INTERVAL_MS, "-1");
+		// Disable the test transaction
+		myProp.put(IntegrityMonitorProperties.TEST_TRANS_INTERVAL, "-1");
+		// Disable the write FPC
+		myProp.put(IntegrityMonitorProperties.WRITE_FPC_INTERVAL, "-1");
+		// Disable the State Audit we will call it directly
+		myProp.put(IntegrityMonitorProperties.STATE_AUDIT_INTERVAL_MS, "-1");
+		// Max interval for use in deciding if a FPC entry is stale in seconds
+		myProp.put(IntegrityMonitorProperties.MAX_FPC_UPDATE_INTERVAL, "120");
+
+		IntegrityMonitor im = makeMonitor(resourceName, myProp);
+		waitStep();
+
+		logger.debug("\nIntegrityMonitorTest: Creating ForwardProgressEntity entries\n\n");
+		// Add resources to put an entry in the forward progress table
+		Date staleDate = new Date(0);
+		ForwardProgressEntity fpe1 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe2 = new ForwardProgressEntity();
+		ForwardProgressEntity fpe3 = new ForwardProgressEntity();
+		fpe1.setFpcCount(0);
+		fpe1.setResourceName("siteA_pap2");
+		fpe2.setFpcCount(0);
+		fpe2.setResourceName("siteB_pap1");
+		fpe3.setFpcCount(0);
+		fpe3.setResourceName("siteB_pap2");
+		logger.debug("\nIntegrityMonitorTest: Creating StateManagementEntity entries\n\n");
+		StateManagementEntity sme1 = new StateManagementEntity();
+		StateManagementEntity sme2 = new StateManagementEntity();
+		StateManagementEntity sme3 = new StateManagementEntity();
+		sme1.setResourceName("siteA_pap2");
+		sme1.setAdminState(StateManagement.UNLOCKED);
+		sme1.setOpState(StateManagement.ENABLED);
+		sme1.setAvailStatus(StateManagement.NULL_VALUE);
+		sme1.setStandbyStatus(StateManagement.NULL_VALUE);
+		sme2.setResourceName("siteB_pap1");
+		sme2.setAdminState(StateManagement.UNLOCKED);
+		sme2.setOpState(StateManagement.ENABLED);
+		sme2.setAvailStatus(StateManagement.NULL_VALUE);
+		sme2.setStandbyStatus(StateManagement.NULL_VALUE);
+		sme3.setResourceName("siteB_pap2");
+		sme3.setAdminState(StateManagement.UNLOCKED);
+		sme3.setOpState(StateManagement.ENABLED);
+		sme3.setAvailStatus(StateManagement.NULL_VALUE);
+		sme3.setStandbyStatus(StateManagement.NULL_VALUE);
+		et = em.getTransaction();
+		et.begin();
+		em.persist(fpe1);
+		em.persist(fpe2);
+		em.persist(fpe3);
+		em.persist(sme1);
+		em.persist(sme2);
+		em.persist(sme3);
+		em.flush();
+		et.commit();
+
+		Query updateQuery = em.createQuery(
+				"UPDATE ForwardProgressEntity f " + "SET f.lastUpdated = :newDate " + "WHERE f.resourceName=:resource");
+		updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+		updateQuery.setParameter("resource", fpe1.getResourceName());
+
+		et = em.getTransaction();
+		et.begin();
+		updateQuery.executeUpdate();
+		et.commit();
+
+		logger.debug("\nIntegrityMonitorTest:testStateAudit Calling im.getAllForwardProgressEntity()\n\n");
+		List<ForwardProgressEntity> fpeList = im.getAllForwardProgressEntity();
+
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:ForwardProgressEntity entries");
+		for (ForwardProgressEntity myFpe : fpeList) {
+			logger.debug("\n    ResourceName: {}" + "\n        LastUpdated: {}", myFpe.getResourceName(),
+					myFpe.getLastUpdated());
+		}
+		logger.debug("\n\n");
+
+		logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+		Query query = em.createQuery("SELECT s FROM StateManagementEntity s");
+		List<?> smeList = query.getResultList();
+
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+		for (Object mySme : smeList) {
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug(
+					"\n    ResourceName: {}" + "\n        AdminState: {}" + "\n        OpState: {}"
+							+ "\n        AvailStatus: {}" + "\n        StandbyStatus: {}",
+					tmpSme.getResourceName(), tmpSme.getAdminState(), tmpSme.getOpState(), tmpSme.getAvailStatus(),
+					tmpSme.getStandbyStatus());
+		}
+		logger.debug("\n\n");
+
+		em.refresh(sme1);
+		assertEquals(StateManagement.ENABLED, sme1.getOpState());
+
+		logger.debug("IntegrityMonitorTest:testStateAudit: calling stateAudit()");
+		im.executeStateAudit();
+		logger.debug("IntegrityMonitorTest:testStateAudit: call to stateAudit() complete");
+
+		logger.debug("\nIntegrityMonitorTest:testStateAudit getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity entries");
+		for (Object mySme : smeList) {
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+			logger.debug(
+					"\n    ResourceName: {}" + "\n        AdminState: {}" + "\n        OpState: {}"
+							+ "\n        AvailStatus: {}" + "\n        StandbyStatus: {}",
+					tmpSme.getResourceName(), tmpSme.getAdminState(), tmpSme.getOpState(), tmpSme.getAvailStatus(),
+					tmpSme.getStandbyStatus());
+		}
+		logger.debug("\n\n");
+
+		em.refresh(sme1);
+		assertEquals(StateManagement.DISABLED, sme1.getOpState());
+
+		// Now let's add sme2 to the mix
+		updateQuery = em.createQuery(
+				"UPDATE ForwardProgressEntity f " + "SET f.lastUpdated = :newDate " + "WHERE f.resourceName=:resource");
+		updateQuery.setParameter("newDate", staleDate, TemporalType.TIMESTAMP);
+		updateQuery.setParameter("resource", fpe2.getResourceName());
+
+		et = em.getTransaction();
+		et.begin();
+		updateQuery.executeUpdate();
+		et.commit();
+
+		// Give it a chance to write the DB and run the audit
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) Running State Audit");
+		waitStep();
+		im.executeStateAudit();
+		waitStep();
+		logger.debug("IntegrityMonitorTest:testStateAudit: (restart4) State Audit complete");
+
+		// Now check its state
+		logger.debug(
+				"\nIntegrityMonitorTest:testStateAudit (restart4) getting list of StateManagementEntity entries\n\n");
+		smeList = query.getResultList();
+
+		logger.debug("\n\n");
+		logger.debug("IntegrityMonitorTest:testStateAudit:StateManagementEntity (restart4) entries");
+		for (Object mySme : smeList) {
+			StateManagementEntity tmpSme = (StateManagementEntity) mySme;
+			em.refresh(tmpSme);
+
+			logger.debug(
+					"\n    (restart4) ResourceName: {}" + "\n        AdminState: {}" + "\n        OpState: {}"
+							+ "\n        AvailStatus: {}" + "\n        StandbyStatus: {}",
+					tmpSme.getResourceName(), tmpSme.getAdminState(), tmpSme.getOpState(), tmpSme.getAvailStatus(),
+					tmpSme.getStandbyStatus());
+		}
+		logger.debug("\n\n");
+
+		em.refresh(sme1);
+		assertEquals(StateManagement.DISABLED, sme1.getOpState());
+
+		em.refresh(sme2);
+		assertEquals(StateManagement.DISABLED, sme2.getOpState());
+
+		logger.debug("\nIntegrityMonitorTest: Exit testStateAudit\n\n");
+		System.out.println("\n\ntestStateAudit: Exit\n\n");
+	}
+
+	private IntegrityMonitor makeMonitor(String resourceName, Properties myProp) throws Exception {
+		IntegrityMonitor.deleteInstance();
+
+		queue = new LinkedBlockingQueue<>();
+
+		IntegrityMonitor im = IntegrityMonitor.getInstance(resourceName, myProp, queue);
+
+		// wait for the monitor thread to start
+		waitStep();
+
+		return im;
+	}
+
+	/**
+	 * Waits for the FPManager to complete another cycle.
+	 * 
+	 * @throws InterruptedException
+	 */
+	private void waitStep() throws InterruptedException {
+		CountDownLatch latch = new CountDownLatch(1);
+		queue.offer(latch);
+		waitLatch(latch);
+	}
+}