Merge "Fix issues blocking election handler thread"

10 files changed, 548 insertions, 141 deletions

diff --git a/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java b/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
index 3ea54eba..70ad3719 100644
--- a/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
+++ b/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
@@ -22,6 +22,9 @@ package org.onap.policy.drools.statemanagement;
 
 import java.util.Observer;
 
+import javax.validation.constraints.NotNull;
+
+import org.onap.policy.common.im.AllSeemsWellException;
 import org.onap.policy.common.im.StandbyStatusException;
 import org.onap.policy.common.im.StateManagement;
 import org.onap.policy.drools.properties.Lockable;
@@ -64,6 +67,9 @@ public interface StateManagementFeatureAPI extends OrderedService, Lockable
 	  public static final String AVAILABLE_STATUS= StateManagement.AVAILABLE_STATUS;
 	  public static final String STANDBY_STATUS  = StateManagement.STANDBY_STATUS;
 	  
+	  static public final Boolean ALLSEEMSWELL = Boolean.TRUE;
+	  static public final Boolean ALLNOTWELL = Boolean.FALSE;
+	  
 	  public static final int SEQ_NUM = 0;
   /**
    * 'FeatureAPI.impl.getList()' returns an ordered list of objects
@@ -73,6 +79,29 @@ public interface StateManagementFeatureAPI extends OrderedService, Lockable
 	new OrderedServiceImpl<>(StateManagementFeatureAPI.class);
 
   /**
+   * ALL SEEMS/NOT WELL
+   * This interface is used to support the concept of All Seems/Not Well.  It provides
+   * a way for client code to indicate to the DroolsPDPIntegrityMonitor that an event 
+   * has occurred which is disabling (or enabling) for the Drools PDP.  The call is
+   * actually implemented in the common modules IntegrityMonitor where it will cause
+   * the testTransaction to fail if any module has set the value ALLNOTWELL, stopping
+   * the forward progress counter and eventually causing the operational state to 
+   * become disabled.
+   * 
+   * ALLSEEMSWELL is passed to the method when the client is healthy
+   * ALLNOTWELL is passed to the method when the client is disabled
+   * 
+   * @param key - This should be a unique identifier for the entity making the call (e.g., class name)
+   * @param asw - This is the indicator of health. See constants: ALLSEEMSWELL or ALLNOTWELL
+   * @param msg - A message is required.  It should indicate why all is not well or a message indicating
+   * that a component has been restored to health (perhaps indicating the problem that has resolved).
+   * @throws IllegalArgumentException
+   * @throws AllSeemsWellException
+   */
+  public void allSeemsWell(@NotNull String key, @NotNull Boolean asw, @NotNull String msg)
+		  throws IllegalArgumentException, AllSeemsWellException;
+  
+  /**
    * This method is called to add an Observer to receive notifications of state changes
    * 
    * @param stateChangeObserver
diff --git a/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java b/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
index e9526eb7..e7f40772 100644
--- a/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
+++ b/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
@@ -35,8 +35,6 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 	// get an instance of logger 
 	private final static Logger  logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);	
 	private DroolsPdpsConnector pdpsConnector;
-	private Object pdpsConnectorLock = new Object();
-	private Object checkUpdateWorkerLock = new Object();
 	private Object checkWaitTimerLock = new Object();
 	private Object designationWaiterLock = new Object();
 	
@@ -49,18 +47,42 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 	private DesignationWaiter designationWaiter;
 	private Timer updateWorker;
 	private Timer waitTimer;
-	private Date updateWorkerLastRunDate;
 	private Date waitTimerLastRunDate;
+	
+	// The interval between checks of the DesignationWaiter to be sure it is running.
 	private int pdpCheckInterval;
+	
+	// The interval between runs of the DesignationWaiter
 	private int pdpUpdateInterval;
+	
 	private volatile boolean isDesignated;
 
 	private String pdpdNowActive;
 	private String pdpdLastActive;
 	
+	private Boolean allSeemsWell=true;
+	
 	private StateManagementFeatureAPI stateManagementFeature;
 	
+	private static boolean isUnitTesting = false;
+	public static void setIsUnitTesting(boolean val){
+		isUnitTesting = val;
+	}
+	private static boolean isStalled = false;
+	public static void setIsStalled(boolean val){
+		isStalled = val;
+	}
+	
 	public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
+		if (pdps == null) {
+			logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
+			throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
+		}
+		if (myPdp == null){
+			logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
+			throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
+		}
+
 		pdpdNowActive = null;
 		pdpdLastActive = null;
 		this.pdpsConnector = pdps;
@@ -71,14 +93,14 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 			pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
 		}catch(Exception e){
 			logger.error
-			("Could not get pdpCheckInterval property. Using default", e);
+			("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
 		}
 		pdpUpdateInterval = 2000;
 		try{
 			pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
 		}catch(Exception e){
 			logger.error
-			("Could not get pdpUpdateInterval property. Using default", e);
+			("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
 		}	
 
 		Date now = new Date();
@@ -90,11 +112,10 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 		// This is the heartbeat 
 		updateWorker = new Timer();
 
-		// Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
+		// Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
 		// NOTE: The first run of the TimerUpdateClass results in myPdp being added to the 
 		// drools droolsPdpEntity table.
 		updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
-		updateWorkerLastRunDate = new Date(nowMs + 100);
 
 		// Create the timer which will run the election algorithm
 		waitTimer = new Timer();
@@ -151,13 +172,11 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 					("DesignatedWaiter.run: Entering");
 				}
 				
-				// just here initially so code still works
-				if (pdpsConnector == null) {
-					waitTimerLastRunDate = new Date();
+				//This is for testing the checkWaitTimer
+				if(isUnitTesting && isStalled){
 					if(logger.isDebugEnabled()){
-						logger.debug("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
+						logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}", isUnitTesting, isStalled);
 					}
-					
 					return;
 				}
 
@@ -168,8 +187,6 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 						("DesignatedWaiter.run: Entering synchronized block");
 					}
 
-					checkUpdateWorkerTimer();
-					
 					//It is possible that multiple PDPs are designated lead.  So, we will make a list of all designated
 					//PDPs and then decide which one really should be designated at the end.
 					ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
@@ -519,6 +536,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 						if(logger.isDebugEnabled()){
 							logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
 						}
+						myPdp.setUpdatedDate(waitTimerLastRunDate);
+						pdpsConnector.update(myPdp);
 						
 						return;
 						
@@ -580,6 +599,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 							logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
 								+ "waitTimerLastRunDate = " + waitTimerLastRunDate);
 						}
+						myPdp.setUpdatedDate(waitTimerLastRunDate);
+						pdpsConnector.update(myPdp);
 
 						return;
 					}
@@ -598,6 +619,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 				}
 				
 				waitTimerLastRunDate = tmpDate;
+				myPdp.setUpdatedDate(waitTimerLastRunDate);
+				pdpsConnector.update(myPdp);
 				
 			}catch(Exception e){
 				logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
@@ -896,92 +919,19 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 					logger.debug("TimerUpdateClass.run: entry");
 				}
 				checkWaitTimer();
-				synchronized(pdpsConnectorLock){
-					
-					myPdp.setUpdatedDate(new Date());
-					/*
-					Redundant with DesignationWaiter and this updates the date every
-					cycle instead of just when the state changes.			
-					if(myPdp.isDesignated()){
-						myPdp.setDesignatedDate(new Date());
-					}
- 					*/					
-					pdpsConnector.update(myPdp);
-					
-					Date tmpDate = new Date();
-					if(logger.isDebugEnabled()){
-						logger.debug("TimerUpdateClass.run: updateWorkerLastRunDate = {}", tmpDate);
-					}
-					
-					updateWorkerLastRunDate = tmpDate;
-				}
-				if(logger.isDebugEnabled()){
-					logger.debug("TimerUpdateClass.run.exit");
-				}
 			}catch(Exception e){
 				logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
 			}
+			if(logger.isDebugEnabled()){
+				logger.debug("TimerUpdateClass.run.exit");
+			}
 		}
 	}
 	@Override
 	public void checkThreadStatus() {
-		checkUpdateWorkerTimer();
 		checkWaitTimer();
 	}
 
-	private void checkUpdateWorkerTimer(){
-		synchronized(checkUpdateWorkerLock){
-			try{
-				if(logger.isDebugEnabled()){
-					logger.debug("checkUpdateWorkerTimer: entry");
-				}
-				Date now = new Date();
-				long nowMs = now.getTime();
-				long updateWorkerMs = updateWorkerLastRunDate.getTime();
-				//give it 2 second cushion
-				if((nowMs - updateWorkerMs)  > pdpCheckInterval + 2000){
-					logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = {} " 
-							+ ", exceeds pdpCheckInterval + 2000 = {} "
-							+ "Will reschedule updateWorker timer",(nowMs - updateWorkerMs), (pdpCheckInterval + 2000));
-
-					try{
-						updateWorker.cancel();
-						// Recalculate the time because this is a synchronized section and the thread could have
-						// been blocked.
-						now = new Date();
-						nowMs = now.getTime();
-						updateWorker = new Timer();
-						// reset the updateWorkerLastRunDate
-						updateWorkerLastRunDate = new Date(nowMs + 100);
-						//execute the first time in 100 ms
-						updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
-						if(logger.isDebugEnabled()){
-							logger.debug("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
-						}
-					}catch(Exception e){
-						logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: ", e);
-						// Recalculate the time because this is a synchronized section and the thread could have
-						// been blocked.
-						now = new Date();
-						nowMs = now.getTime();
-						updateWorker = new Timer();
-						updateWorkerLastRunDate = new Date(nowMs + 100);
-						updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
-						if(logger.isDebugEnabled()){
-							logger.debug("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
-						}
-					}
-
-				}
-				if(logger.isDebugEnabled()){
-					logger.debug("checkUpdateWorkerTimer: exit");
-				}
-			}catch(Exception e){
-				logger.error("checkUpdateWorkerTimer: caught unexpected exception: ", e);
-			}
-		}
-	}
-
 	private void checkWaitTimer(){
 		synchronized(checkWaitTimerLock){
 			try{
@@ -994,38 +944,24 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
 
 				//give it 10 times leeway  
 				if((nowMs - waitTimerMs)  > 10*pdpUpdateInterval){
-					logger.error("checkWaitTimer: nowMs - waitTimerMs = {}" 
-							+ ", exceeds 10* pdpUpdateInterval = {}"
-							+ "Will reschedule waitTimer timer", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
-
-					try{
-						// Recalculate since the thread could have been stalled on the synchronize()
-						nowMs = (new Date()).getTime();
-						// Time to the start of the next pdpUpdateInterval multiple
-						long startMs = getDWaiterStartMs();
-						waitTimer.cancel();
-						designationWaiter = new DesignationWaiter();
-						waitTimer = new Timer();
-						waitTimerLastRunDate = new Date(nowMs + startMs);
-						waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
-						if(logger.isDebugEnabled()){
-							logger.debug("checkWaitTimer: Scheduling waitTimer timer to start in {} ms", startMs);
-						}
-					}catch(Exception e){
-						logger.error("checkWaitTimer: Caught unexpected Exception: ", e);
-						// Recalculate since the thread could have been stalled on the synchronize()
-						nowMs = (new Date()).getTime();
-						// Time to the start of the next pdpUpdateInterval multiple
-						long startMs = getDWaiterStartMs();
-						designationWaiter = new DesignationWaiter();
-						waitTimer = new Timer();
-						waitTimerLastRunDate = new Date(nowMs + startMs);
-						waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
+					if(allSeemsWell){
+						allSeemsWell = false;
 						if(logger.isDebugEnabled()){
-							logger.debug("checkWaitTimer: Scheduling waitTimer timer in {} ms", startMs);
+							logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
 						}
+						stateManagementFeature.allSeemsWell(this.getClass().getName(), 
+								StateManagementFeatureAPI.ALLNOTWELL,
+								"DesignationWaiter/ElectionHandler has STALLED");
 					}
-
+					logger.error("checkWaitTimer: nowMs - waitTimerMs = {}" 
+							+ ", exceeds 10* pdpUpdateInterval = {}"
+							+ " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
+				}else if(!allSeemsWell){
+					allSeemsWell = true;
+					stateManagementFeature.allSeemsWell(this.getClass().getName(), 
+							StateManagementFeatureAPI.ALLSEEMSWELL,
+							"DesignationWaiter/ElectionHandler has RESUMED");
+					logger.info("DesignationWaiter/ElectionHandler has RESUMED");
 				}
 				if(logger.isDebugEnabled()){
 					logger.debug("checkWaitTimer: exit");
diff --git a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java
new file mode 100644
index 00000000..0318bed6
--- /dev/null
+++ b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java
@@ -0,0 +1,316 @@
+/*
+ * ============LICENSE_START=======================================================
+ * feature-active-standby-management
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.controller.test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Date;
+import java.util.Properties;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.EntityTransaction;
+import javax.persistence.Persistence;
+
+import org.apache.commons.lang3.time.DateUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onap.policy.common.im.StateManagement;
+import org.onap.policy.drools.activestandby.ActiveStandbyFeatureAPI;
+import org.onap.policy.drools.activestandby.ActiveStandbyProperties;
+import org.onap.policy.drools.activestandby.DroolsPdpEntity;
+import org.onap.policy.drools.activestandby.DroolsPdpImpl;
+import org.onap.policy.drools.activestandby.DroolsPdpsConnector;
+import org.onap.policy.drools.activestandby.DroolsPdpsElectionHandler;
+import org.onap.policy.drools.activestandby.JpaDroolsPdpsConnector;
+import org.onap.policy.drools.core.PolicySessionFeatureAPI;
+import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Testing the allSeemsWell interface to verify that it correctly affects the 
+ * operational state.
+ */ 
+ 
+public class AllSeemsWellTest {
+	private static final Logger  logger = LoggerFactory.getLogger(AllSeemsWellTest.class);
+	/*
+	 * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every 1 seconds, starting 
+	 * at the start of the next multiple of pdpUpdateInterval, but with a minimum of 5 sec cushion
+	 * to ensure that we wait for the DesignationWaiter to do its job, before 
+	 * checking the results. Add a few seconds for safety
+	 */ 
+	 
+	long sleepTime = 10000;
+	
+	/*
+	 * DroolsPdpsElectionHandler runs every 1 seconds, so it takes 10 seconds for the
+	 * checkWaitTimer() method to time out and call allSeemsWell which then requires
+	 * the forward progress counter to go stale which should add an additional 5 sec.
+	 */
+	 
+	long stalledElectionHandlerSleepTime = 15000;
+	
+	/*
+	 * As soon as the election hander successfully runs, it will resume the forward progress.
+	 * If the election handler runs ever 1 sec and test transaction is run every 1 sec and
+	 * then fpc is written every 1 sec and then the fpc is checked every 2 sec, that could
+	 * take a total of 5 sec to recognize the resumption of progress.  So, add 1 for safety.
+	 */
+	long resumedElectionHandlerSleepTime = 6000;
+	
+	private static EntityManagerFactory emfx;
+	private static EntityManagerFactory emfd;
+	private static EntityManager emx;
+	private static EntityManager emd;
+	private static EntityTransaction et;
+	
+	private final String configDir = "src/test/resources/asw";
+
+	/*
+	 * See the IntegrityMonitor.getJmxUrl() method for the rationale behind this jmx related processing.
+	 */
+	 
+	@BeforeClass
+	public static void setUpClass() throws Exception {
+		
+		String userDir = System.getProperty("user.dir");
+		logger.debug("setUpClass: userDir={}", userDir);
+		System.setProperty("com.sun.management.jmxremote.port", "9980");
+		System.setProperty("com.sun.management.jmxremote.authenticate","false");
+	}
+
+	@AfterClass
+	public static void tearDownClass() throws Exception {
+	}
+
+	@Before
+	public void setUp() throws Exception {
+		//Create teh data access for xaml db
+		Properties stateManagementProperties = new Properties();
+		stateManagementProperties.load(new FileInputStream(new File(
+				configDir + "/feature-state-management.properties")));
+
+		emfx = Persistence.createEntityManagerFactory("junitXacmlPU", stateManagementProperties);
+
+		// Create an entity manager to use the DB
+		emx = emfx.createEntityManager();
+		
+		//Create the data access for drools db
+		Properties activeStandbyProperties = new Properties();
+		activeStandbyProperties.load(new FileInputStream(new File(
+				configDir + "/feature-active-standby-management.properties")));
+
+		emfd = Persistence.createEntityManagerFactory("junitDroolsPU", activeStandbyProperties);
+
+		// Create an entity manager to use the DB
+		emd = emfd.createEntityManager();
+		
+		DroolsPdpsElectionHandler.setIsUnitTesting(true);
+	}
+
+	@After
+	public void tearDown() throws Exception {
+				
+	}
+	
+	public void cleanXacmlDb(){
+		et = emx.getTransaction();
+		
+		et.begin();
+		// Make sure we leave the DB clean
+		emx.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+		emx.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+		emx.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+		emx.flush();
+		et.commit();
+	}
+	
+	public void cleanDroolsDb(){
+		et = emd.getTransaction();
+		
+		et.begin();
+		// Make sure we leave the DB clean
+		emd.createQuery("DELETE FROM DroolsPdpEntity").executeUpdate();
+		emd.flush();
+		et.commit();
+	}
+	
+
+	// Tests hot standby when there is only one PDP.
+	 
+	//@Ignore
+	@Test
+	public void testAllSeemsWell() throws Exception {
+	
+		logger.debug("\n\ntestAllSeemsWell: Entering\n\n");
+		cleanXacmlDb();
+		cleanDroolsDb();
+		
+		logger.debug("testAllSeemsWell: Reading stateManagementProperties");
+		Properties stateManagementProperties = new Properties();
+		stateManagementProperties.load(new FileInputStream(new File(
+				configDir + "/feature-state-management.properties")));
+
+		logger.debug("testAllSeemsWell: Creating emfXacml");
+		EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
+				"junitXacmlPU", stateManagementProperties);
+		
+		logger.debug("testAllSeemsWell: Reading activeStandbyProperties");
+		Properties activeStandbyProperties = new Properties();
+		activeStandbyProperties.load(new FileInputStream(new File(
+				configDir + "/feature-active-standby-management.properties")));
+		String thisPdpId = activeStandbyProperties
+				.getProperty(ActiveStandbyProperties.NODE_NAME);
+
+		logger.debug("testAllSeemsWell: Creating emfDrools");
+		EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
+				"junitDroolsPU", activeStandbyProperties);
+		
+		DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
+		
+		logger.debug("testAllSeemsWell: Cleaning up tables");
+		conn.deleteAllPdps();
+					
+		/*
+		 * Insert this PDP as not designated.  Initial standby state will be 
+		 * either null or cold standby.   Demoting should transit state to
+		 * hot standby.
+		 */
+		 
+		logger.debug("testAllSeemsWell: Inserting PDP={} as not designated", thisPdpId);
+		Date yesterday = DateUtils.addDays(new Date(), -1);
+		DroolsPdpImpl pdp = new DroolsPdpImpl(thisPdpId, false, 4, yesterday);
+		conn.insertPdp(pdp);
+		DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
+		logger.debug("testAllSeemsWell: After insertion, PDP={} has DESIGNATED={}",
+				thisPdpId, droolsPdpEntity.isDesignated());
+		assertTrue(droolsPdpEntity.isDesignated() == false);
+		
+		logger.debug("testAllSeemsWell: Instantiating stateManagement object");
+		StateManagement sm = new StateManagement(emfXacml, "dummy");
+		sm.deleteAllStateManagementEntities();
+		
+		
+		// Now we want to create a StateManagementFeature and initialize it.  It will be
+		// discovered by the ActiveStandbyFeature when the election handler initializes.
+
+		StateManagementFeatureAPI smf = null;
+		for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
+		{
+			((PolicySessionFeatureAPI) feature).globalInit(null, configDir);
+			smf = feature;
+			logger.debug("testAllSeemsWell stateManagementFeature.getResourceName(): {}", smf.getResourceName());
+			break;
+		}
+		if(smf == null){
+			logger.error("testAllSeemsWell failed to initialize.  "
+					+ "Unable to get instance of StateManagementFeatureAPI "
+					+ "with resourceID: {}", thisPdpId);
+			logger.debug("testAllSeemsWell failed to initialize.  "
+					+ "Unable to get instance of StateManagementFeatureAPI "
+					+ "with resourceID: {}", thisPdpId);
+		}
+		
+		// Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
+		// that has been created.
+		ActiveStandbyFeatureAPI activeStandbyFeature = null;
+		for (ActiveStandbyFeatureAPI feature : ActiveStandbyFeatureAPI.impl.getList())
+		{
+			((PolicySessionFeatureAPI) feature).globalInit(null, configDir);
+			activeStandbyFeature = feature;
+			logger.debug("testAllSeemsWell activeStandbyFeature.getResourceName(): {}", activeStandbyFeature.getResourceName());
+			break;
+		}
+		if(activeStandbyFeature == null){
+			logger.error("testAllSeemsWell failed to initialize.  "
+					+ "Unable to get instance of ActiveStandbyFeatureAPI "
+					+ "with resourceID: {}", thisPdpId);
+			logger.debug("testAllSeemsWell failed to initialize.  "
+					+ "Unable to get instance of ActiveStandbyFeatureAPI "
+					+ "with resourceID: {}", thisPdpId);
+		}
+		
+
+		logger.debug("testAllSeemsWell: Demoting PDP={}", thisPdpId);
+		// demoting should cause state to transit to hotstandby
+		smf.demote();
+		
+				
+		logger.debug("testAllSeemsWell: Sleeping {} ms, to allow JpaDroolsPdpsConnector "
+				+ "time to check droolspdpentity table", sleepTime);
+		Thread.sleep(sleepTime);
+		
+		
+		// Verify that this formerly un-designated PDP in HOT_STANDBY is now designated and providing service.
+		 
+		droolsPdpEntity = conn.getPdp(thisPdpId);
+		logger.debug("testAllSeemsWell: After sm.demote() invoked, DESIGNATED= {} "
+				+ "for PDP= {}", droolsPdpEntity.isDesignated(), thisPdpId);
+		assertTrue(droolsPdpEntity.isDesignated() == true);
+		String standbyStatus = smf.getStandbyStatus(thisPdpId);
+		logger.debug("testAllSeemsWell: After demotion, PDP= {} "
+				+ "has standbyStatus= {}", thisPdpId, standbyStatus);
+		assertTrue(standbyStatus != null  &&  standbyStatus.equals(StateManagement.PROVIDING_SERVICE));
+		
+		//Now we want to stall the election handler and see the if AllSeemsWell will make the
+		//standbystatus = coldstandby
+		
+		DroolsPdpsElectionHandler.setIsStalled(true);
+		
+		logger.debug("testAllSeemsWell: Sleeping {} ms, to allow checkWaitTimer to recognize "
+				+ "the election handler has stalled and for the testTransaction to fail to "
+				+ "increment forward progress and for the lack of forward progress to be recognized.",
+				stalledElectionHandlerSleepTime);
+
+
+		//It takes 10x the update interval (1 sec) before the watcher will declare the election handler dead
+		//and that just stops forward progress counter.  So, the fp monitor must then run to determine
+		//if the fpc has stalled.  That will take about another 5 sec.
+		Thread.sleep(stalledElectionHandlerSleepTime);
+		
+		logger.debug("testAllSeemsWell: After isStalled=true, PDP= {} "
+				+ "has standbyStatus= {}", thisPdpId, smf.getStandbyStatus(thisPdpId));
+		
+		assertTrue(smf.getStandbyStatus().equals(StateManagement.COLD_STANDBY));
+		
+		//Now lets resume the election handler
+		DroolsPdpsElectionHandler.setIsStalled(false);
+		
+		Thread.sleep(resumedElectionHandlerSleepTime);
+		
+		logger.debug("testAllSeemsWell: After isStalled=false, PDP= {} "
+				+ "has standbyStatus= {}", thisPdpId, smf.getStandbyStatus(thisPdpId));
+		
+		assertTrue(smf.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE));
+
+		//resumedElectionHandlerSleepTime = 5000;
+		logger.debug("\n\ntestAllSeemsWell: Exiting\n\n");
+
+	}
+}
diff --git a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
index 4f7469cc..876cd99a 100644
--- a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
+++ b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
@@ -38,6 +38,7 @@ import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.onap.policy.common.im.AdministrativeStateException;
 import org.onap.policy.common.im.IntegrityMonitor;
@@ -71,23 +72,23 @@ import org.slf4j.LoggerFactory;
 public class StandbyStateManagementTest {
 	private static final Logger  logger = LoggerFactory.getLogger(StandbyStateManagementTest.class);
 	/*
-	 * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every ten seconds, starting 
-	 * at ten seconds after the minute boundary (e.g. 13:05:10). So, an 80 second sleep should be 
-	 * sufficient to ensure that we wait for the DesignationWaiter to do its job, before 
-	 * checking the results.
+	 * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every 1 seconds, starting 
+	 * at the start of the next multiple of pdpUpdateInterval, but with a minimum of 5 sec cushion
+	 * to ensure that we wait for the DesignationWaiter to do its job, before 
+	 * checking the results. Add a few seconds for safety
 	 */ 
 	 
-	long sleepTime = 80000;
+	long sleepTime = 10000;
 	
 	/*
-	 * DroolsPdpsElectionHandler runs every ten seconds, so a 15 second sleep should be 
+	 * DroolsPdpsElectionHandler runs every 1 seconds, so a 6 second sleep should be 
 	 * plenty to ensure it has time to re-promote this PDP.
 	 */
 	 
-	long electionWaitSleepTime = 15000;
+	long electionWaitSleepTime = 6000;
 	
 	/*
-	 * Sleep 5 seconds after each test to allow interrupt (shutdown) recovery.
+	 * Sleep 1 seconds after each test to allow interrupt (shutdown) recovery.
 	 */
 	 
 	long interruptRecoveryTime = 5000;
diff --git a/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties b/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties
new file mode 100644
index 00000000..f0711e6c
--- /dev/null
+++ b/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties
@@ -0,0 +1,39 @@
+###
+# ============LICENSE_START=======================================================
+# feature-active-standby-management
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+
+# DB properties
+javax.persistence.jdbc.driver = org.h2.Driver
+javax.persistence.jdbc.url  = jdbc:h2:file:./sql/activestandbymanagement
+javax.persistence.jdbc.user = sa
+javax.persistence.jdbc.password =
+
+# Must be unique across the system
+resource.name=pdp1
+# Name of the site in which this node is hosted 
+site_name=pdp_1
+
+# Needed by DroolsPdpsElectionHandler
+pdp.checkInterval=1500
+pdp.updateInterval=1000
+
+# Need long timeout, because testTransaction is only run every 1 seconds.
+pdp.timeout=3000
+#how long do we wait for the pdp table to populate on initial startup
+pdp.initialWait=1000
\ No newline at end of file
diff --git a/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties b/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties
new file mode 100644
index 00000000..2629c63d
--- /dev/null
+++ b/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties
@@ -0,0 +1,81 @@
+###
+# ============LICENSE_START=======================================================
+# feature-active-standby-management
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+
+# DB properties
+javax.persistence.jdbc.driver = org.h2.Driver
+javax.persistence.jdbc.url  = jdbc:h2:file:./sql/statemanagement
+javax.persistence.jdbc.user = sa
+javax.persistence.jdbc.password =
+
+# DroolsPDPIntegrityMonitor Properties
+
+http.server.services=TEST
+http.server.services.TEST.host=0.0.0.0
+http.server.services.TEST.port=9981
+#These properties will default to the following if no other values are provided:
+# http.server.services.TEST.restClasses=org.onap.policy.drools.statemanagement.IntegrityMonitorRestManager
+# http.server.services.TEST.managed=false
+# http.server.services.TEST.swagger=true
+
+#IntegrityMonitor Properties
+
+# Must be unique across the system
+resource.name=pdp1
+# Name of the site in which this node is hosted 
+site_name = pdp_1
+# How often in sec the forward progress monitor checks for forward progress
+fp_monitor_interval = 2
+# Failed counter threshold before failover 
+failed_counter_threshold = 1
+# Interval between test transactions when no traffic seconds
+test_trans_interval = 1
+# Interval between writes of the FPC to the DB seconds 
+write_fpc_interval = 1
+# Node type Note: Make sure you don't leave any trailing spaces, or you'll get an 'invalid node type' error! 
+node_type = pdp_drools
+# Dependency groups are groups of resources upon which a node operational state is dependent upon. 
+# Each group is a comma-separated list of resource names and groups are separated by a semicolon.  For example:
+# dependency_groups=site_1.astra_1,site_1.astra_2;site_1.brms_1,site_1.brms_2;site_1.logparser_1;site_1.pypdp_1
+dependency_groups=
+# When set to true, dependent health checks are performed by using JMX to invoke test() on the dependent.
+# The default false is to use state checks for health.
+test_via_jmx=true
+# This is the max number of seconds beyond which a non incrementing FPC is considered a failure
+max_fpc_update_interval=5
+# Run the state audit every 60 seconds (60000 ms).  The state audit finds stale DB entries in the 
+# forwardprogressentity table and marks the node as disabled/failed in the statemanagemententity 
+# table. NOTE! It will only run on nodes that have a standbystatus = providingservice.
+# A value of <= 0 will turn off the state audit.
+state_audit_interval_ms= -1
+# The refresh state audit is run every (default) 10 minutes (600000 ms) to clean up any state corruption in the 
+# DB statemanagemententity table. It only refreshes the DB state entry for the local node.  That is, it does not
+# refresh the state of any other nodes.  A value <= 0 will turn the audit off. Any other value will override 
+# the default of 600000 ms.
+refresh_state_audit_interval_ms=-1
+
+
+# Repository audit properties
+# Flag to control the execution of the subsystemTest for the Nexus Maven repository
+repository.audit.is.active=false
+repository.audit.ignore.errors=true
+
+# DB Audit Properties
+# Flag to control the execution of the subsystemTest for the Database
+db.audit.is.active=false
diff --git a/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties b/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
index bbae5d98..827d2e17 100644
--- a/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
+++ b/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
@@ -32,8 +32,7 @@ site_name=pdp_1
 # Needed by DroolsPdpsElectionHandler
 pdp.checkInterval=1500
 pdp.updateInterval=1000
-#pdp.timeout=3000
 # Need long timeout, because testTransaction is only run every 10 seconds.
-pdp.timeout=15000
+pdp.timeout=3000
 #how long do we wait for the pdp table to populate on initial startup
-pdp.initialWait=20000
\ No newline at end of file
+pdp.initialWait=1000
\ No newline at end of file
diff --git a/feature-active-standby-management/src/test/resources/feature-state-management.properties b/feature-active-standby-management/src/test/resources/feature-state-management.properties
index 7856d251..3dd88473 100644
--- a/feature-active-standby-management/src/test/resources/feature-state-management.properties
+++ b/feature-active-standby-management/src/test/resources/feature-state-management.properties
@@ -40,13 +40,13 @@ resource.name=pdp1
 # Name of the site in which this node is hosted 
 site_name = pdp_1
 # Forward Progress Monitor update interval seconds
-fp_monitor_interval = 30
+fp_monitor_interval = 2
 # Failed counter threshold before failover 
-failed_counter_threshold = 3
+failed_counter_threshold = 1
 # Interval between test transactions when no traffic seconds
-test_trans_interval = 10
+test_trans_interval = 1
 # Interval between writes of the FPC to the DB seconds 
-write_fpc_interval = 5
+write_fpc_interval = 1
 # Node type Note: Make sure you don't leave any trailing spaces, or you'll get an 'invalid node type' error! 
 node_type = pdp_drools
 # Dependency groups are groups of resources upon which a node operational state is dependent upon. 
@@ -57,17 +57,17 @@ dependency_groups=
 # The default false is to use state checks for health.
 test_via_jmx=true
 # This is the max number of seconds beyond which a non incrementing FPC is considered a failure
-max_fpc_update_interval=120
+max_fpc_update_interval=5
 # Run the state audit every 60 seconds (60000 ms).  The state audit finds stale DB entries in the 
 # forwardprogressentity table and marks the node as disabled/failed in the statemanagemententity 
 # table. NOTE! It will only run on nodes that have a standbystatus = providingservice.
 # A value of <= 0 will turn off the state audit.
-state_audit_interval_ms=60000
+state_audit_interval_ms=-1
 # The refresh state audit is run every (default) 10 minutes (600000 ms) to clean up any state corruption in the 
 # DB statemanagemententity table. It only refreshes the DB state entry for the local node.  That is, it does not
 # refresh the state of any other nodes.  A value <= 0 will turn the audit off. Any other value will override 
 # the default of 600000 ms.
-refresh_state_audit_interval_ms=600000
+refresh_state_audit_interval_ms=-1
 
 
 # Repository audit properties
diff --git a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
index f599e3dc..688631bb 100644
--- a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
+++ b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
@@ -20,11 +20,8 @@
 
 package org.onap.policy.drools.statemanagement;
 
-import org.onap.policy.drools.statemanagement.StateManagementProperties;
-
 import java.util.ArrayList;
 import java.util.Properties;
-
 import org.onap.policy.common.im.IntegrityMonitor;
 import org.onap.policy.common.im.IntegrityMonitorException;
 import org.onap.policy.drools.http.server.HttpServletServer;
@@ -82,6 +79,7 @@ public class DroolsPDPIntegrityMonitor extends IntegrityMonitor
 		  logger.info(msg);
 	  }
   }
+  
   /**
    * Static initialization -- create Drools Integrity Monitor, and
    * an HTTP server to handle REST 'test' requests
@@ -221,7 +219,6 @@ public class DroolsPDPIntegrityMonitor extends IntegrityMonitor
 								testPort, e);
 		throw e;
 	}
-	
 	logger.info("init: Exiting and returning DroolsPDPIntegrityMonitor");
 	return im;
   }
diff --git a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
index 66d806be..a34d4f98 100644
--- a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
+++ b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.util.Observer;
 import java.util.Properties;
 
+import org.onap.policy.common.im.AllSeemsWellException;
 import org.onap.policy.common.im.StateManagement;
 import org.onap.policy.drools.core.PolicySessionFeatureAPI;
 import org.onap.policy.drools.features.PolicyEngineFeatureAPI;
@@ -263,4 +264,12 @@ public class StateManagementFeature implements StateManagementFeatureAPI,
 			logger.error("initializeProperties", e1);
 		}
 	}
+
+	@Override
+	public void allSeemsWell(String key, Boolean asw, String msg)
+			throws IllegalArgumentException, AllSeemsWellException {
+
+		droolsPdpIntegrityMonitor.allSeemsWell(key, asw, msg);
+		
+	}
 }