aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJorge Hernandez <jh1730@att.com>2017-12-21 18:13:51 +0000
committerGerrit Code Review <gerrit@onap.org>2017-12-21 18:13:51 +0000
commit3db97797677739506be089c28365aff91695bdc2 (patch)
tree55848544ed389fb05176e2e78f93ca9aab026074
parentd117c4bab1f05d49467dadfadb409d9fc538f890 (diff)
parentbc8c8286645f74753d175eee7ca62d989555c96c (diff)
Merge "Fix issues blocking election handler thread"
-rw-r--r--api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java29
-rw-r--r--feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java174
-rw-r--r--feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java316
-rw-r--r--feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java17
-rw-r--r--feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties39
-rw-r--r--feature-active-standby-management/src/test/resources/asw/feature-state-management.properties81
-rw-r--r--feature-active-standby-management/src/test/resources/feature-active-standby-management.properties5
-rw-r--r--feature-active-standby-management/src/test/resources/feature-state-management.properties14
-rw-r--r--feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java5
-rw-r--r--feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java9
10 files changed, 548 insertions, 141 deletions
diff --git a/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java b/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
index 3ea54eba..70ad3719 100644
--- a/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
+++ b/api-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeatureAPI.java
@@ -22,6 +22,9 @@ package org.onap.policy.drools.statemanagement;
import java.util.Observer;
+import javax.validation.constraints.NotNull;
+
+import org.onap.policy.common.im.AllSeemsWellException;
import org.onap.policy.common.im.StandbyStatusException;
import org.onap.policy.common.im.StateManagement;
import org.onap.policy.drools.properties.Lockable;
@@ -64,6 +67,9 @@ public interface StateManagementFeatureAPI extends OrderedService, Lockable
public static final String AVAILABLE_STATUS= StateManagement.AVAILABLE_STATUS;
public static final String STANDBY_STATUS = StateManagement.STANDBY_STATUS;
+ static public final Boolean ALLSEEMSWELL = Boolean.TRUE;
+ static public final Boolean ALLNOTWELL = Boolean.FALSE;
+
public static final int SEQ_NUM = 0;
/**
* 'FeatureAPI.impl.getList()' returns an ordered list of objects
@@ -73,6 +79,29 @@ public interface StateManagementFeatureAPI extends OrderedService, Lockable
new OrderedServiceImpl<>(StateManagementFeatureAPI.class);
/**
+ * ALL SEEMS/NOT WELL
+ * This interface is used to support the concept of All Seems/Not Well. It provides
+ * a way for client code to indicate to the DroolsPDPIntegrityMonitor that an event
+ * has occurred which is disabling (or enabling) for the Drools PDP. The call is
+ * actually implemented in the common modules IntegrityMonitor where it will cause
+ * the testTransaction to fail if any module has set the value ALLNOTWELL, stopping
+ * the forward progress counter and eventually causing the operational state to
+ * become disabled.
+ *
+ * ALLSEEMSWELL is passed to the method when the client is healthy
+ * ALLNOTWELL is passed to the method when the client is disabled
+ *
+ * @param key - This should be a unique identifier for the entity making the call (e.g., class name)
+ * @param asw - This is the indicator of health. See constants: ALLSEEMSWELL or ALLNOTWELL
+ * @param msg - A message is required. It should indicate why all is not well or a message indicating
+ * that a component has been restored to health (perhaps indicating the problem that has resolved).
+ * @throws IllegalArgumentException
+ * @throws AllSeemsWellException
+ */
+ public void allSeemsWell(@NotNull String key, @NotNull Boolean asw, @NotNull String msg)
+ throws IllegalArgumentException, AllSeemsWellException;
+
+ /**
* This method is called to add an Observer to receive notifications of state changes
*
* @param stateChangeObserver
diff --git a/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java b/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
index e9526eb7..e7f40772 100644
--- a/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
+++ b/feature-active-standby-management/src/main/java/org/onap/policy/drools/activestandby/DroolsPdpsElectionHandler.java
@@ -35,8 +35,6 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
// get an instance of logger
private final static Logger logger = LoggerFactory.getLogger(DroolsPdpsElectionHandler.class);
private DroolsPdpsConnector pdpsConnector;
- private Object pdpsConnectorLock = new Object();
- private Object checkUpdateWorkerLock = new Object();
private Object checkWaitTimerLock = new Object();
private Object designationWaiterLock = new Object();
@@ -49,18 +47,42 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
private DesignationWaiter designationWaiter;
private Timer updateWorker;
private Timer waitTimer;
- private Date updateWorkerLastRunDate;
private Date waitTimerLastRunDate;
+
+ // The interval between checks of the DesignationWaiter to be sure it is running.
private int pdpCheckInterval;
+
+ // The interval between runs of the DesignationWaiter
private int pdpUpdateInterval;
+
private volatile boolean isDesignated;
private String pdpdNowActive;
private String pdpdLastActive;
+ private Boolean allSeemsWell=true;
+
private StateManagementFeatureAPI stateManagementFeature;
+ private static boolean isUnitTesting = false;
+ public static void setIsUnitTesting(boolean val){
+ isUnitTesting = val;
+ }
+ private static boolean isStalled = false;
+ public static void setIsStalled(boolean val){
+ isStalled = val;
+ }
+
public DroolsPdpsElectionHandler(DroolsPdpsConnector pdps, DroolsPdp myPdp){
+ if (pdps == null) {
+ logger.error("DroolsPdpsElectinHandler(): pdpsConnector==null");
+ throw new IllegalArgumentException("DroolsPdpsElectinHandler(): pdpsConnector==null");
+ }
+ if (myPdp == null){
+ logger.error("DroolsPdpsElectinHandler(): droolsPdp==null");
+ throw new IllegalArgumentException("DroolsPdpsElectinHandler(): DroolsPdp==null");
+ }
+
pdpdNowActive = null;
pdpdLastActive = null;
this.pdpsConnector = pdps;
@@ -71,14 +93,14 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
pdpCheckInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_CHECK_INVERVAL));
}catch(Exception e){
logger.error
- ("Could not get pdpCheckInterval property. Using default", e);
+ ("Could not get pdpCheckInterval property. Using default {}",pdpCheckInterval, e);
}
pdpUpdateInterval = 2000;
try{
pdpUpdateInterval = Integer.parseInt(ActiveStandbyProperties.getProperty(ActiveStandbyProperties.PDP_UPDATE_INTERVAL));
}catch(Exception e){
logger.error
- ("Could not get pdpUpdateInterval property. Using default", e);
+ ("Could not get pdpUpdateInterval property. Using default {} ", pdpUpdateInterval, e);
}
Date now = new Date();
@@ -90,11 +112,10 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
// This is the heartbeat
updateWorker = new Timer();
- // Schedule the heartbeat to start in 100 ms and run at pdpCheckInterval ms thereafter
+ // Schedule the TimerUpdateClass to run at 100 ms and run at pdpCheckInterval ms thereafter
// NOTE: The first run of the TimerUpdateClass results in myPdp being added to the
// drools droolsPdpEntity table.
updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
- updateWorkerLastRunDate = new Date(nowMs + 100);
// Create the timer which will run the election algorithm
waitTimer = new Timer();
@@ -151,13 +172,11 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
("DesignatedWaiter.run: Entering");
}
- // just here initially so code still works
- if (pdpsConnector == null) {
- waitTimerLastRunDate = new Date();
+ //This is for testing the checkWaitTimer
+ if(isUnitTesting && isStalled){
if(logger.isDebugEnabled()){
- logger.debug("DesignatedWaiter.run (pdpsConnector==null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
+ logger.debug("DesignatedWaiter.run: isUnitTesting = {} isStalled = {}", isUnitTesting, isStalled);
}
-
return;
}
@@ -168,8 +187,6 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
("DesignatedWaiter.run: Entering synchronized block");
}
- checkUpdateWorkerTimer();
-
//It is possible that multiple PDPs are designated lead. So, we will make a list of all designated
//PDPs and then decide which one really should be designated at the end.
ArrayList<DroolsPdp> listOfDesignated = new ArrayList<DroolsPdp>();
@@ -519,6 +536,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
if(logger.isDebugEnabled()){
logger.debug("DesignatedWaiter.run (designatedPdp == null) waitTimerLastRunDate = {}", waitTimerLastRunDate);
}
+ myPdp.setUpdatedDate(waitTimerLastRunDate);
+ pdpsConnector.update(myPdp);
return;
@@ -580,6 +599,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
logger.debug("DesignatedWaiter.run (designatedPdp.getPdpId().equals(myPdp.getPdpId())) "
+ "waitTimerLastRunDate = " + waitTimerLastRunDate);
}
+ myPdp.setUpdatedDate(waitTimerLastRunDate);
+ pdpsConnector.update(myPdp);
return;
}
@@ -598,6 +619,8 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
}
waitTimerLastRunDate = tmpDate;
+ myPdp.setUpdatedDate(waitTimerLastRunDate);
+ pdpsConnector.update(myPdp);
}catch(Exception e){
logger.error("DesignatedWaiter.run caught an unexpected exception: ", e);
@@ -896,92 +919,19 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
logger.debug("TimerUpdateClass.run: entry");
}
checkWaitTimer();
- synchronized(pdpsConnectorLock){
-
- myPdp.setUpdatedDate(new Date());
- /*
- Redundant with DesignationWaiter and this updates the date every
- cycle instead of just when the state changes.
- if(myPdp.isDesignated()){
- myPdp.setDesignatedDate(new Date());
- }
- */
- pdpsConnector.update(myPdp);
-
- Date tmpDate = new Date();
- if(logger.isDebugEnabled()){
- logger.debug("TimerUpdateClass.run: updateWorkerLastRunDate = {}", tmpDate);
- }
-
- updateWorkerLastRunDate = tmpDate;
- }
- if(logger.isDebugEnabled()){
- logger.debug("TimerUpdateClass.run.exit");
- }
}catch(Exception e){
logger.error("TimerUpdateClass.run caught an unexpected exception: ", e);
}
+ if(logger.isDebugEnabled()){
+ logger.debug("TimerUpdateClass.run.exit");
+ }
}
}
@Override
public void checkThreadStatus() {
- checkUpdateWorkerTimer();
checkWaitTimer();
}
- private void checkUpdateWorkerTimer(){
- synchronized(checkUpdateWorkerLock){
- try{
- if(logger.isDebugEnabled()){
- logger.debug("checkUpdateWorkerTimer: entry");
- }
- Date now = new Date();
- long nowMs = now.getTime();
- long updateWorkerMs = updateWorkerLastRunDate.getTime();
- //give it 2 second cushion
- if((nowMs - updateWorkerMs) > pdpCheckInterval + 2000){
- logger.error("checkUpdateWorkerTimer: nowMs - updateWorkerMs = {} "
- + ", exceeds pdpCheckInterval + 2000 = {} "
- + "Will reschedule updateWorker timer",(nowMs - updateWorkerMs), (pdpCheckInterval + 2000));
-
- try{
- updateWorker.cancel();
- // Recalculate the time because this is a synchronized section and the thread could have
- // been blocked.
- now = new Date();
- nowMs = now.getTime();
- updateWorker = new Timer();
- // reset the updateWorkerLastRunDate
- updateWorkerLastRunDate = new Date(nowMs + 100);
- //execute the first time in 100 ms
- updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
- if(logger.isDebugEnabled()){
- logger.debug("checkUpdateWorkerTimer: Scheduling updateWorker timer to start in 100 ms ");
- }
- }catch(Exception e){
- logger.error("checkUpdateWorkerTimer: Caught unexpected Exception: ", e);
- // Recalculate the time because this is a synchronized section and the thread could have
- // been blocked.
- now = new Date();
- nowMs = now.getTime();
- updateWorker = new Timer();
- updateWorkerLastRunDate = new Date(nowMs + 100);
- updateWorker.scheduleAtFixedRate(new TimerUpdateClass(), 100, pdpCheckInterval);
- if(logger.isDebugEnabled()){
- logger.debug("checkUpdateWorkerTimer: Attempting to schedule updateWorker timer in 100 ms");
- }
- }
-
- }
- if(logger.isDebugEnabled()){
- logger.debug("checkUpdateWorkerTimer: exit");
- }
- }catch(Exception e){
- logger.error("checkUpdateWorkerTimer: caught unexpected exception: ", e);
- }
- }
- }
-
private void checkWaitTimer(){
synchronized(checkWaitTimerLock){
try{
@@ -994,38 +944,24 @@ public class DroolsPdpsElectionHandler implements ThreadRunningChecker {
//give it 10 times leeway
if((nowMs - waitTimerMs) > 10*pdpUpdateInterval){
- logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
- + ", exceeds 10* pdpUpdateInterval = {}"
- + "Will reschedule waitTimer timer", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
-
- try{
- // Recalculate since the thread could have been stalled on the synchronize()
- nowMs = (new Date()).getTime();
- // Time to the start of the next pdpUpdateInterval multiple
- long startMs = getDWaiterStartMs();
- waitTimer.cancel();
- designationWaiter = new DesignationWaiter();
- waitTimer = new Timer();
- waitTimerLastRunDate = new Date(nowMs + startMs);
- waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
- if(logger.isDebugEnabled()){
- logger.debug("checkWaitTimer: Scheduling waitTimer timer to start in {} ms", startMs);
- }
- }catch(Exception e){
- logger.error("checkWaitTimer: Caught unexpected Exception: ", e);
- // Recalculate since the thread could have been stalled on the synchronize()
- nowMs = (new Date()).getTime();
- // Time to the start of the next pdpUpdateInterval multiple
- long startMs = getDWaiterStartMs();
- designationWaiter = new DesignationWaiter();
- waitTimer = new Timer();
- waitTimerLastRunDate = new Date(nowMs + startMs);
- waitTimer.scheduleAtFixedRate(designationWaiter, startMs, pdpUpdateInterval);
+ if(allSeemsWell){
+ allSeemsWell = false;
if(logger.isDebugEnabled()){
- logger.debug("checkWaitTimer: Scheduling waitTimer timer in {} ms", startMs);
+ logger.debug("checkWaitTimer: calling allSeemsWell with ALLNOTWELL param");
}
+ stateManagementFeature.allSeemsWell(this.getClass().getName(),
+ StateManagementFeatureAPI.ALLNOTWELL,
+ "DesignationWaiter/ElectionHandler has STALLED");
}
-
+ logger.error("checkWaitTimer: nowMs - waitTimerMs = {}"
+ + ", exceeds 10* pdpUpdateInterval = {}"
+ + " DesignationWaiter is STALLED!", (nowMs - waitTimerMs), (10*pdpUpdateInterval));
+ }else if(!allSeemsWell){
+ allSeemsWell = true;
+ stateManagementFeature.allSeemsWell(this.getClass().getName(),
+ StateManagementFeatureAPI.ALLSEEMSWELL,
+ "DesignationWaiter/ElectionHandler has RESUMED");
+ logger.info("DesignationWaiter/ElectionHandler has RESUMED");
}
if(logger.isDebugEnabled()){
logger.debug("checkWaitTimer: exit");
diff --git a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java
new file mode 100644
index 00000000..0318bed6
--- /dev/null
+++ b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/AllSeemsWellTest.java
@@ -0,0 +1,316 @@
+/*
+ * ============LICENSE_START=======================================================
+ * feature-active-standby-management
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.controller.test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Date;
+import java.util.Properties;
+
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.EntityTransaction;
+import javax.persistence.Persistence;
+
+import org.apache.commons.lang3.time.DateUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onap.policy.common.im.StateManagement;
+import org.onap.policy.drools.activestandby.ActiveStandbyFeatureAPI;
+import org.onap.policy.drools.activestandby.ActiveStandbyProperties;
+import org.onap.policy.drools.activestandby.DroolsPdpEntity;
+import org.onap.policy.drools.activestandby.DroolsPdpImpl;
+import org.onap.policy.drools.activestandby.DroolsPdpsConnector;
+import org.onap.policy.drools.activestandby.DroolsPdpsElectionHandler;
+import org.onap.policy.drools.activestandby.JpaDroolsPdpsConnector;
+import org.onap.policy.drools.core.PolicySessionFeatureAPI;
+import org.onap.policy.drools.statemanagement.StateManagementFeatureAPI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Testing the allSeemsWell interface to verify that it correctly affects the
+ * operational state.
+ */
+
+public class AllSeemsWellTest {
+ private static final Logger logger = LoggerFactory.getLogger(AllSeemsWellTest.class);
+ /*
+ * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every 1 seconds, starting
+ * at the start of the next multiple of pdpUpdateInterval, but with a minimum of 5 sec cushion
+ * to ensure that we wait for the DesignationWaiter to do its job, before
+ * checking the results. Add a few seconds for safety
+ */
+
+ long sleepTime = 10000;
+
+ /*
+ * DroolsPdpsElectionHandler runs every 1 seconds, so it takes 10 seconds for the
+ * checkWaitTimer() method to time out and call allSeemsWell which then requires
+ * the forward progress counter to go stale which should add an additional 5 sec.
+ */
+
+ long stalledElectionHandlerSleepTime = 15000;
+
+ /*
+ * As soon as the election hander successfully runs, it will resume the forward progress.
+ * If the election handler runs ever 1 sec and test transaction is run every 1 sec and
+ * then fpc is written every 1 sec and then the fpc is checked every 2 sec, that could
+ * take a total of 5 sec to recognize the resumption of progress. So, add 1 for safety.
+ */
+ long resumedElectionHandlerSleepTime = 6000;
+
+ private static EntityManagerFactory emfx;
+ private static EntityManagerFactory emfd;
+ private static EntityManager emx;
+ private static EntityManager emd;
+ private static EntityTransaction et;
+
+ private final String configDir = "src/test/resources/asw";
+
+ /*
+ * See the IntegrityMonitor.getJmxUrl() method for the rationale behind this jmx related processing.
+ */
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+
+ String userDir = System.getProperty("user.dir");
+ logger.debug("setUpClass: userDir={}", userDir);
+ System.setProperty("com.sun.management.jmxremote.port", "9980");
+ System.setProperty("com.sun.management.jmxremote.authenticate","false");
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ //Create teh data access for xaml db
+ Properties stateManagementProperties = new Properties();
+ stateManagementProperties.load(new FileInputStream(new File(
+ configDir + "/feature-state-management.properties")));
+
+ emfx = Persistence.createEntityManagerFactory("junitXacmlPU", stateManagementProperties);
+
+ // Create an entity manager to use the DB
+ emx = emfx.createEntityManager();
+
+ //Create the data access for drools db
+ Properties activeStandbyProperties = new Properties();
+ activeStandbyProperties.load(new FileInputStream(new File(
+ configDir + "/feature-active-standby-management.properties")));
+
+ emfd = Persistence.createEntityManagerFactory("junitDroolsPU", activeStandbyProperties);
+
+ // Create an entity manager to use the DB
+ emd = emfd.createEntityManager();
+
+ DroolsPdpsElectionHandler.setIsUnitTesting(true);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+
+ }
+
+ public void cleanXacmlDb(){
+ et = emx.getTransaction();
+
+ et.begin();
+ // Make sure we leave the DB clean
+ emx.createQuery("DELETE FROM StateManagementEntity").executeUpdate();
+ emx.createQuery("DELETE FROM ResourceRegistrationEntity").executeUpdate();
+ emx.createQuery("DELETE FROM ForwardProgressEntity").executeUpdate();
+ emx.flush();
+ et.commit();
+ }
+
+ public void cleanDroolsDb(){
+ et = emd.getTransaction();
+
+ et.begin();
+ // Make sure we leave the DB clean
+ emd.createQuery("DELETE FROM DroolsPdpEntity").executeUpdate();
+ emd.flush();
+ et.commit();
+ }
+
+
+ // Tests hot standby when there is only one PDP.
+
+ //@Ignore
+ @Test
+ public void testAllSeemsWell() throws Exception {
+
+ logger.debug("\n\ntestAllSeemsWell: Entering\n\n");
+ cleanXacmlDb();
+ cleanDroolsDb();
+
+ logger.debug("testAllSeemsWell: Reading stateManagementProperties");
+ Properties stateManagementProperties = new Properties();
+ stateManagementProperties.load(new FileInputStream(new File(
+ configDir + "/feature-state-management.properties")));
+
+ logger.debug("testAllSeemsWell: Creating emfXacml");
+ EntityManagerFactory emfXacml = Persistence.createEntityManagerFactory(
+ "junitXacmlPU", stateManagementProperties);
+
+ logger.debug("testAllSeemsWell: Reading activeStandbyProperties");
+ Properties activeStandbyProperties = new Properties();
+ activeStandbyProperties.load(new FileInputStream(new File(
+ configDir + "/feature-active-standby-management.properties")));
+ String thisPdpId = activeStandbyProperties
+ .getProperty(ActiveStandbyProperties.NODE_NAME);
+
+ logger.debug("testAllSeemsWell: Creating emfDrools");
+ EntityManagerFactory emfDrools = Persistence.createEntityManagerFactory(
+ "junitDroolsPU", activeStandbyProperties);
+
+ DroolsPdpsConnector conn = new JpaDroolsPdpsConnector(emfDrools);
+
+ logger.debug("testAllSeemsWell: Cleaning up tables");
+ conn.deleteAllPdps();
+
+ /*
+ * Insert this PDP as not designated. Initial standby state will be
+ * either null or cold standby. Demoting should transit state to
+ * hot standby.
+ */
+
+ logger.debug("testAllSeemsWell: Inserting PDP={} as not designated", thisPdpId);
+ Date yesterday = DateUtils.addDays(new Date(), -1);
+ DroolsPdpImpl pdp = new DroolsPdpImpl(thisPdpId, false, 4, yesterday);
+ conn.insertPdp(pdp);
+ DroolsPdpEntity droolsPdpEntity = conn.getPdp(thisPdpId);
+ logger.debug("testAllSeemsWell: After insertion, PDP={} has DESIGNATED={}",
+ thisPdpId, droolsPdpEntity.isDesignated());
+ assertTrue(droolsPdpEntity.isDesignated() == false);
+
+ logger.debug("testAllSeemsWell: Instantiating stateManagement object");
+ StateManagement sm = new StateManagement(emfXacml, "dummy");
+ sm.deleteAllStateManagementEntities();
+
+
+ // Now we want to create a StateManagementFeature and initialize it. It will be
+ // discovered by the ActiveStandbyFeature when the election handler initializes.
+
+ StateManagementFeatureAPI smf = null;
+ for (StateManagementFeatureAPI feature : StateManagementFeatureAPI.impl.getList())
+ {
+ ((PolicySessionFeatureAPI) feature).globalInit(null, configDir);
+ smf = feature;
+ logger.debug("testAllSeemsWell stateManagementFeature.getResourceName(): {}", smf.getResourceName());
+ break;
+ }
+ if(smf == null){
+ logger.error("testAllSeemsWell failed to initialize. "
+ + "Unable to get instance of StateManagementFeatureAPI "
+ + "with resourceID: {}", thisPdpId);
+ logger.debug("testAllSeemsWell failed to initialize. "
+ + "Unable to get instance of StateManagementFeatureAPI "
+ + "with resourceID: {}", thisPdpId);
+ }
+
+ // Create an ActiveStandbyFeature and initialize it. It will discover the StateManagementFeature
+ // that has been created.
+ ActiveStandbyFeatureAPI activeStandbyFeature = null;
+ for (ActiveStandbyFeatureAPI feature : ActiveStandbyFeatureAPI.impl.getList())
+ {
+ ((PolicySessionFeatureAPI) feature).globalInit(null, configDir);
+ activeStandbyFeature = feature;
+ logger.debug("testAllSeemsWell activeStandbyFeature.getResourceName(): {}", activeStandbyFeature.getResourceName());
+ break;
+ }
+ if(activeStandbyFeature == null){
+ logger.error("testAllSeemsWell failed to initialize. "
+ + "Unable to get instance of ActiveStandbyFeatureAPI "
+ + "with resourceID: {}", thisPdpId);
+ logger.debug("testAllSeemsWell failed to initialize. "
+ + "Unable to get instance of ActiveStandbyFeatureAPI "
+ + "with resourceID: {}", thisPdpId);
+ }
+
+
+ logger.debug("testAllSeemsWell: Demoting PDP={}", thisPdpId);
+ // demoting should cause state to transit to hotstandby
+ smf.demote();
+
+
+ logger.debug("testAllSeemsWell: Sleeping {} ms, to allow JpaDroolsPdpsConnector "
+ + "time to check droolspdpentity table", sleepTime);
+ Thread.sleep(sleepTime);
+
+
+ // Verify that this formerly un-designated PDP in HOT_STANDBY is now designated and providing service.
+
+ droolsPdpEntity = conn.getPdp(thisPdpId);
+ logger.debug("testAllSeemsWell: After sm.demote() invoked, DESIGNATED= {} "
+ + "for PDP= {}", droolsPdpEntity.isDesignated(), thisPdpId);
+ assertTrue(droolsPdpEntity.isDesignated() == true);
+ String standbyStatus = smf.getStandbyStatus(thisPdpId);
+ logger.debug("testAllSeemsWell: After demotion, PDP= {} "
+ + "has standbyStatus= {}", thisPdpId, standbyStatus);
+ assertTrue(standbyStatus != null && standbyStatus.equals(StateManagement.PROVIDING_SERVICE));
+
+ //Now we want to stall the election handler and see the if AllSeemsWell will make the
+ //standbystatus = coldstandby
+
+ DroolsPdpsElectionHandler.setIsStalled(true);
+
+ logger.debug("testAllSeemsWell: Sleeping {} ms, to allow checkWaitTimer to recognize "
+ + "the election handler has stalled and for the testTransaction to fail to "
+ + "increment forward progress and for the lack of forward progress to be recognized.",
+ stalledElectionHandlerSleepTime);
+
+
+ //It takes 10x the update interval (1 sec) before the watcher will declare the election handler dead
+ //and that just stops forward progress counter. So, the fp monitor must then run to determine
+ //if the fpc has stalled. That will take about another 5 sec.
+ Thread.sleep(stalledElectionHandlerSleepTime);
+
+ logger.debug("testAllSeemsWell: After isStalled=true, PDP= {} "
+ + "has standbyStatus= {}", thisPdpId, smf.getStandbyStatus(thisPdpId));
+
+ assertTrue(smf.getStandbyStatus().equals(StateManagement.COLD_STANDBY));
+
+ //Now lets resume the election handler
+ DroolsPdpsElectionHandler.setIsStalled(false);
+
+ Thread.sleep(resumedElectionHandlerSleepTime);
+
+ logger.debug("testAllSeemsWell: After isStalled=false, PDP= {} "
+ + "has standbyStatus= {}", thisPdpId, smf.getStandbyStatus(thisPdpId));
+
+ assertTrue(smf.getStandbyStatus().equals(StateManagement.PROVIDING_SERVICE));
+
+ //resumedElectionHandlerSleepTime = 5000;
+ logger.debug("\n\ntestAllSeemsWell: Exiting\n\n");
+
+ }
+}
diff --git a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
index 4f7469cc..876cd99a 100644
--- a/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
+++ b/feature-active-standby-management/src/test/java/org/onap/policy/drools/controller/test/StandbyStateManagementTest.java
@@ -38,6 +38,7 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import org.onap.policy.common.im.AdministrativeStateException;
import org.onap.policy.common.im.IntegrityMonitor;
@@ -71,23 +72,23 @@ import org.slf4j.LoggerFactory;
public class StandbyStateManagementTest {
private static final Logger logger = LoggerFactory.getLogger(StandbyStateManagementTest.class);
/*
- * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every ten seconds, starting
- * at ten seconds after the minute boundary (e.g. 13:05:10). So, an 80 second sleep should be
- * sufficient to ensure that we wait for the DesignationWaiter to do its job, before
- * checking the results.
+ * Currently, the DroolsPdpsElectionHandler.DesignationWaiter is invoked every 1 seconds, starting
+ * at the start of the next multiple of pdpUpdateInterval, but with a minimum of 5 sec cushion
+ * to ensure that we wait for the DesignationWaiter to do its job, before
+ * checking the results. Add a few seconds for safety
*/
- long sleepTime = 80000;
+ long sleepTime = 10000;
/*
- * DroolsPdpsElectionHandler runs every ten seconds, so a 15 second sleep should be
+ * DroolsPdpsElectionHandler runs every 1 seconds, so a 6 second sleep should be
* plenty to ensure it has time to re-promote this PDP.
*/
- long electionWaitSleepTime = 15000;
+ long electionWaitSleepTime = 6000;
/*
- * Sleep 5 seconds after each test to allow interrupt (shutdown) recovery.
+ * Sleep 1 seconds after each test to allow interrupt (shutdown) recovery.
*/
long interruptRecoveryTime = 5000;
diff --git a/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties b/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties
new file mode 100644
index 00000000..f0711e6c
--- /dev/null
+++ b/feature-active-standby-management/src/test/resources/asw/feature-active-standby-management.properties
@@ -0,0 +1,39 @@
+###
+# ============LICENSE_START=======================================================
+# feature-active-standby-management
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+
+# DB properties
+javax.persistence.jdbc.driver = org.h2.Driver
+javax.persistence.jdbc.url = jdbc:h2:file:./sql/activestandbymanagement
+javax.persistence.jdbc.user = sa
+javax.persistence.jdbc.password =
+
+# Must be unique across the system
+resource.name=pdp1
+# Name of the site in which this node is hosted
+site_name=pdp_1
+
+# Needed by DroolsPdpsElectionHandler
+pdp.checkInterval=1500
+pdp.updateInterval=1000
+
+# Need long timeout, because testTransaction is only run every 1 seconds.
+pdp.timeout=3000
+#how long do we wait for the pdp table to populate on initial startup
+pdp.initialWait=1000 \ No newline at end of file
diff --git a/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties b/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties
new file mode 100644
index 00000000..2629c63d
--- /dev/null
+++ b/feature-active-standby-management/src/test/resources/asw/feature-state-management.properties
@@ -0,0 +1,81 @@
+###
+# ============LICENSE_START=======================================================
+# feature-active-standby-management
+# ================================================================================
+# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+
+# DB properties
+javax.persistence.jdbc.driver = org.h2.Driver
+javax.persistence.jdbc.url = jdbc:h2:file:./sql/statemanagement
+javax.persistence.jdbc.user = sa
+javax.persistence.jdbc.password =
+
+# DroolsPDPIntegrityMonitor Properties
+
+http.server.services=TEST
+http.server.services.TEST.host=0.0.0.0
+http.server.services.TEST.port=9981
+#These properties will default to the following if no other values are provided:
+# http.server.services.TEST.restClasses=org.onap.policy.drools.statemanagement.IntegrityMonitorRestManager
+# http.server.services.TEST.managed=false
+# http.server.services.TEST.swagger=true
+
+#IntegrityMonitor Properties
+
+# Must be unique across the system
+resource.name=pdp1
+# Name of the site in which this node is hosted
+site_name = pdp_1
+# How often in sec the forward progress monitor checks for forward progress
+fp_monitor_interval = 2
+# Failed counter threshold before failover
+failed_counter_threshold = 1
+# Interval between test transactions when no traffic seconds
+test_trans_interval = 1
+# Interval between writes of the FPC to the DB seconds
+write_fpc_interval = 1
+# Node type Note: Make sure you don't leave any trailing spaces, or you'll get an 'invalid node type' error!
+node_type = pdp_drools
+# Dependency groups are groups of resources upon which a node operational state is dependent upon.
+# Each group is a comma-separated list of resource names and groups are separated by a semicolon. For example:
+# dependency_groups=site_1.astra_1,site_1.astra_2;site_1.brms_1,site_1.brms_2;site_1.logparser_1;site_1.pypdp_1
+dependency_groups=
+# When set to true, dependent health checks are performed by using JMX to invoke test() on the dependent.
+# The default false is to use state checks for health.
+test_via_jmx=true
+# This is the max number of seconds beyond which a non incrementing FPC is considered a failure
+max_fpc_update_interval=5
+# Run the state audit every 60 seconds (60000 ms). The state audit finds stale DB entries in the
+# forwardprogressentity table and marks the node as disabled/failed in the statemanagemententity
+# table. NOTE! It will only run on nodes that have a standbystatus = providingservice.
+# A value of <= 0 will turn off the state audit.
+state_audit_interval_ms= -1
+# The refresh state audit is run every (default) 10 minutes (600000 ms) to clean up any state corruption in the
+# DB statemanagemententity table. It only refreshes the DB state entry for the local node. That is, it does not
+# refresh the state of any other nodes. A value <= 0 will turn the audit off. Any other value will override
+# the default of 600000 ms.
+refresh_state_audit_interval_ms=-1
+
+
+# Repository audit properties
+# Flag to control the execution of the subsystemTest for the Nexus Maven repository
+repository.audit.is.active=false
+repository.audit.ignore.errors=true
+
+# DB Audit Properties
+# Flag to control the execution of the subsystemTest for the Database
+db.audit.is.active=false
diff --git a/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties b/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
index bbae5d98..827d2e17 100644
--- a/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
+++ b/feature-active-standby-management/src/test/resources/feature-active-standby-management.properties
@@ -32,8 +32,7 @@ site_name=pdp_1
# Needed by DroolsPdpsElectionHandler
pdp.checkInterval=1500
pdp.updateInterval=1000
-#pdp.timeout=3000
# Need long timeout, because testTransaction is only run every 10 seconds.
-pdp.timeout=15000
+pdp.timeout=3000
#how long do we wait for the pdp table to populate on initial startup
-pdp.initialWait=20000 \ No newline at end of file
+pdp.initialWait=1000 \ No newline at end of file
diff --git a/feature-active-standby-management/src/test/resources/feature-state-management.properties b/feature-active-standby-management/src/test/resources/feature-state-management.properties
index 7856d251..3dd88473 100644
--- a/feature-active-standby-management/src/test/resources/feature-state-management.properties
+++ b/feature-active-standby-management/src/test/resources/feature-state-management.properties
@@ -40,13 +40,13 @@ resource.name=pdp1
# Name of the site in which this node is hosted
site_name = pdp_1
# Forward Progress Monitor update interval seconds
-fp_monitor_interval = 30
+fp_monitor_interval = 2
# Failed counter threshold before failover
-failed_counter_threshold = 3
+failed_counter_threshold = 1
# Interval between test transactions when no traffic seconds
-test_trans_interval = 10
+test_trans_interval = 1
# Interval between writes of the FPC to the DB seconds
-write_fpc_interval = 5
+write_fpc_interval = 1
# Node type Note: Make sure you don't leave any trailing spaces, or you'll get an 'invalid node type' error!
node_type = pdp_drools
# Dependency groups are groups of resources upon which a node operational state is dependent upon.
@@ -57,17 +57,17 @@ dependency_groups=
# The default false is to use state checks for health.
test_via_jmx=true
# This is the max number of seconds beyond which a non incrementing FPC is considered a failure
-max_fpc_update_interval=120
+max_fpc_update_interval=5
# Run the state audit every 60 seconds (60000 ms). The state audit finds stale DB entries in the
# forwardprogressentity table and marks the node as disabled/failed in the statemanagemententity
# table. NOTE! It will only run on nodes that have a standbystatus = providingservice.
# A value of <= 0 will turn off the state audit.
-state_audit_interval_ms=60000
+state_audit_interval_ms=-1
# The refresh state audit is run every (default) 10 minutes (600000 ms) to clean up any state corruption in the
# DB statemanagemententity table. It only refreshes the DB state entry for the local node. That is, it does not
# refresh the state of any other nodes. A value <= 0 will turn the audit off. Any other value will override
# the default of 600000 ms.
-refresh_state_audit_interval_ms=600000
+refresh_state_audit_interval_ms=-1
# Repository audit properties
diff --git a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
index f599e3dc..688631bb 100644
--- a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
+++ b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/DroolsPDPIntegrityMonitor.java
@@ -20,11 +20,8 @@
package org.onap.policy.drools.statemanagement;
-import org.onap.policy.drools.statemanagement.StateManagementProperties;
-
import java.util.ArrayList;
import java.util.Properties;
-
import org.onap.policy.common.im.IntegrityMonitor;
import org.onap.policy.common.im.IntegrityMonitorException;
import org.onap.policy.drools.http.server.HttpServletServer;
@@ -82,6 +79,7 @@ public class DroolsPDPIntegrityMonitor extends IntegrityMonitor
logger.info(msg);
}
}
+
/**
* Static initialization -- create Drools Integrity Monitor, and
* an HTTP server to handle REST 'test' requests
@@ -221,7 +219,6 @@ public class DroolsPDPIntegrityMonitor extends IntegrityMonitor
testPort, e);
throw e;
}
-
logger.info("init: Exiting and returning DroolsPDPIntegrityMonitor");
return im;
}
diff --git a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
index 66d806be..a34d4f98 100644
--- a/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
+++ b/feature-state-management/src/main/java/org/onap/policy/drools/statemanagement/StateManagementFeature.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.Observer;
import java.util.Properties;
+import org.onap.policy.common.im.AllSeemsWellException;
import org.onap.policy.common.im.StateManagement;
import org.onap.policy.drools.core.PolicySessionFeatureAPI;
import org.onap.policy.drools.features.PolicyEngineFeatureAPI;
@@ -263,4 +264,12 @@ public class StateManagementFeature implements StateManagementFeatureAPI,
logger.error("initializeProperties", e1);
}
}
+
+ @Override
+ public void allSeemsWell(String key, Boolean asw, String msg)
+ throws IllegalArgumentException, AllSeemsWellException {
+
+ droolsPdpIntegrityMonitor.allSeemsWell(key, asw, msg);
+
+ }
}