Add PDP heart beat expiration timer

Added heart beat interval to the PDP-UPDATE message sent in response
to a heart beat message received from a PDP.
Added timers to detect missing heart beats and remove the PDP from
the DB - PdpTracker.
Modified current heart beat listener to update PdpTracker when a
heart beat is received.
Allow 3 missed heart beats instead of 2.

Change-Id: I81621fefbe494e0c4d6f0b9767b00b2a9dd398d8
Issue-ID: POLICY-1795
Signed-off-by: jrh3 <jrh3@att.com>

9 files changed, 271 insertions, 3 deletions

diff --git a/main/src/test/java/org/onap/policy/pap/main/comm/PdpTrackerTest.java b/main/src/test/java/org/onap/policy/pap/main/comm/PdpTrackerTest.java
new file mode 100644
index 00000000..19684a71
--- /dev/null
+++ b/main/src/test/java/org/onap/policy/pap/main/comm/PdpTrackerTest.java
@@ -0,0 +1,212 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP PAP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.pap.main.comm;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Consumer;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.onap.policy.common.utils.coder.StandardCoder;
+import org.onap.policy.common.utils.resources.ResourceUtils;
+import org.onap.policy.models.base.PfModelException;
+import org.onap.policy.models.pdp.concepts.PdpGroup;
+import org.onap.policy.models.pdp.concepts.PdpGroups;
+import org.onap.policy.models.provider.PolicyModelsProvider;
+import org.onap.policy.pap.main.PolicyModelsProviderFactoryWrapper;
+import org.onap.policy.pap.main.PolicyPapRuntimeException;
+import org.onap.policy.pap.main.comm.TimerManager.Timer;
+
+public class PdpTrackerTest {
+    private static final String PDP1 = "pdp1";
+    private static final String PDP2 = "pdp2";
+
+    private PdpTracker tracker;
+    private PdpTracker.PdpTrackerBuilder builder;
+
+    private Object modifyLock;
+
+    @Captor
+    private ArgumentCaptor<Consumer<String>> handlerCaptor;
+
+    @Mock
+    private PdpModifyRequestMap requestMap;
+
+    @Mock
+    private TimerManager timers;
+
+    @Mock
+    private PolicyModelsProviderFactoryWrapper daoFactory;
+
+    @Mock
+    private PolicyModelsProvider dao;
+
+    @Mock
+    private Timer timer1;
+
+    @Mock
+    private Timer timer2;
+
+    /**
+     * Sets up.
+     *
+     * @throws Exception if an error occurs
+     */
+    @Before
+    public void setUp() throws Exception {
+        MockitoAnnotations.initMocks(this);
+
+        modifyLock = new Object();
+
+        builder = PdpTracker.builder().daoFactory(daoFactory).modifyLock(modifyLock).requestMap(requestMap)
+                        .timers(timers);
+
+        when(daoFactory.create()).thenReturn(dao);
+
+        when(dao.getPdpGroups(null)).thenReturn(Collections.emptyList());
+
+        when(timers.register(eq(PDP1), any())).thenReturn(timer1);
+        when(timers.register(eq(PDP2), any())).thenReturn(timer2);
+
+        tracker = builder.build();
+    }
+
+    @Test
+    public void testPdpTracker() throws Exception {
+        // verify that PDPs were loaded
+        verify(dao).getPdpGroups(null);
+    }
+
+    @Test
+    public void testBuilderToString() throws Exception {
+        assertNotNull(builder.toString());
+    }
+
+    @Test
+    public void testPdpTracker_MissingRequestMap() throws Exception {
+        assertThatThrownBy(() -> builder.requestMap(null).build()).isInstanceOf(NullPointerException.class);
+    }
+
+    @Test
+    public void testPdpTracker_MissingModifyLock() throws Exception {
+        assertThatThrownBy(() -> builder.modifyLock(null).build()).isInstanceOf(NullPointerException.class);
+    }
+
+    @Test
+    public void testPdpTracker_MissingTimers() throws Exception {
+        assertThatThrownBy(() -> builder.timers(null).build()).isInstanceOf(NullPointerException.class);
+    }
+
+    @Test
+    public void testPdpTracker_MissingDaoFactory() throws Exception {
+        assertThatThrownBy(() -> builder.daoFactory(null).build()).isInstanceOf(NullPointerException.class);
+    }
+
+    @Test
+    public void testLoadPdps_testLoadPdpsFromGroup() throws Exception {
+        // arrange for DAO to return a couple of groups
+        String groupsJson = ResourceUtils.getResourceAsString("comm/PdpTracker.json");
+        List<PdpGroup> groups = new StandardCoder().decode(groupsJson, PdpGroups.class).getGroups();
+        when(dao.getPdpGroups(null)).thenReturn(groups);
+
+        tracker = builder.build();
+
+        // verify that all PDPs were registered
+        verify(timers).register(eq("pdp-A"), any());
+        verify(timers).register(eq("pdp-B"), any());
+        verify(timers).register(eq("pdp-C"), any());
+        verify(timers).register(eq("pdp-D"), any());
+    }
+
+    @Test
+    public void testLoadPdps_DaoException() throws Exception {
+        // arrange for DAO to throw an exception
+        PfModelException ex = mock(PfModelException.class);
+        when(daoFactory.create()).thenThrow(ex);
+
+        assertThatThrownBy(() -> builder.build()).isInstanceOf(PolicyPapRuntimeException.class).hasCause(ex);
+    }
+
+    @Test
+    public void testAdd() {
+        tracker.add(PDP1);
+        verify(timers).register(eq(PDP1), any());
+        verify(timer1, never()).cancel();
+
+        tracker.add(PDP2);
+        verify(timers).register(eq(PDP2), any());
+        verify(timer1, never()).cancel();
+        verify(timer2, never()).cancel();
+
+        // re-add PDP1 - old timer should be canceled and a new timer added
+        Timer timer3 = mock(Timer.class);
+        when(timers.register(eq(PDP1), any())).thenReturn(timer3);
+        tracker.add(PDP1);
+        verify(timer1).cancel();
+        verify(timer2, never()).cancel();
+        verify(timer3, never()).cancel();
+    }
+
+    @Test
+    public void testHandleTimeout() throws Exception {
+        tracker.add(PDP1);
+        tracker.add(PDP2);
+
+        verify(timers).register(eq(PDP1), handlerCaptor.capture());
+
+        handlerCaptor.getValue().accept(PDP1);
+
+        verify(requestMap).removeFromGroups(PDP1);
+
+        // now we'll re-add PDP1 - the original timer should not be canceled
+        Timer timer3 = mock(Timer.class);
+        when(timers.register(eq(PDP1), any())).thenReturn(timer3);
+        tracker.add(PDP1);
+        verify(timer1, never()).cancel();
+    }
+
+    @Test
+    public void testHandleTimeout_MapException() throws Exception {
+        tracker.add(PDP1);
+
+        verify(timers).register(eq(PDP1), handlerCaptor.capture());
+
+        // arrange for request map to throw an exception
+        PfModelException ex = mock(PfModelException.class);
+        when(requestMap.removeFromGroups(PDP1)).thenThrow(ex);
+
+        // exception should be caught, but not re-thrown
+        handlerCaptor.getValue().accept(PDP1);
+    }
+}
diff --git a/main/src/test/java/org/onap/policy/pap/main/parameters/TestPdpParameters.java b/main/src/test/java/org/onap/policy/pap/main/parameters/TestPdpParameters.java
index eb9a6e8b..1474bbfb 100644
--- a/main/src/test/java/org/onap/policy/pap/main/parameters/TestPdpParameters.java
+++ b/main/src/test/java/org/onap/policy/pap/main/parameters/TestPdpParameters.java
@@ -46,6 +46,8 @@ public class TestPdpParameters {
         PdpStateChangeParameters state = params.getStateChangeParameters();
         assertNotNull(state);
         assertEquals(5, state.getMaxWaitMs());
+
+        assertEquals(6L, params.getHeartBeatMs());
     }
 
     @Test
@@ -58,6 +60,13 @@ public class TestPdpParameters {
         assertNull(result.getResult());
         assertTrue(result.isValid());
 
+        // invalid heart beat
+        json2 = json.replaceFirst(": 6", ": 0");
+        result = coder.decode(json2, PapParameterGroup.class).getPdpParameters().validate();
+        assertFalse(result.isValid());
+        assertTrue(result.getResult().contains(
+                        "field 'heartBeatMs' type 'long' value '0' INVALID, must be >= 1".replace('\'', '"')));
+
         // no update params
         json2 = testData.nullifyField(json, "updateParameters");
         result = coder.decode(json2, PapParameterGroup.class).getPdpParameters().validate();
@@ -66,7 +75,7 @@ public class TestPdpParameters {
         assertTrue(result.getResult().contains("is null"));
 
         // invalid update params
-        json2 = json.replaceFirst("2", "-2");
+        json2 = json.replaceFirst(": 2", ": -2");
         result = coder.decode(json2, PapParameterGroup.class).getPdpParameters().validate();
         assertFalse(result.isValid());
         assertTrue(result.getResult().contains("parameter group 'PdpUpdateParameters'".replace('\'', '"')));
@@ -79,7 +88,7 @@ public class TestPdpParameters {
         assertFalse(result.isValid());
 
         // invalid state-change params
-        json2 = json.replaceFirst("5", "-5");
+        json2 = json.replaceFirst(": 5", ": -5");
         result = coder.decode(json2, PapParameterGroup.class).getPdpParameters().validate();
         assertFalse(result.isValid());
         assertTrue(result.getResult().contains("parameter group 'PdpStateChangeParameters'".replace('\'', '"')));
diff --git a/main/src/test/resources/comm/PdpTracker.json b/main/src/test/resources/comm/PdpTracker.json
new file mode 100644
index 00000000..5c8942d5
--- /dev/null
+++ b/main/src/test/resources/comm/PdpTracker.json
@@ -0,0 +1,41 @@
+{
+    "groups": [
+        {
+            "name": "group-X",
+            "pdpSubgroups": [
+                {
+                    "pdpType": "apex",
+                    "pdpInstances": [
+                        {
+                            "instanceId": "pdp-A"
+                        },
+                        {
+                            "instanceId": "pdp-B"
+                        }
+                    ]
+                },
+                {
+                    "pdpType": "drools",
+                    "pdpInstances": [
+                        {
+                            "instanceId": "pdp-C"
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "group-Y",
+            "pdpSubgroups": [
+                {
+                    "pdpType": "apex",
+                    "pdpInstances": [
+                        {
+                            "instanceId": "pdp-D"
+                        }
+                    ]
+                }
+            ]
+        } 
+    ]
+}
diff --git a/main/src/test/resources/e2e/PapConfigParameters.json b/main/src/test/resources/e2e/PapConfigParameters.json
index 4e085f76..871077c5 100644
--- a/main/src/test/resources/e2e/PapConfigParameters.json
+++ b/main/src/test/resources/e2e/PapConfigParameters.json
@@ -8,6 +8,7 @@
         "https": true
     },
     "pdpParameters": {
+        "heartBeatMs": 5000,
         "updateParameters": {
             "maxRetryCount": 0,
             "maxWaitMs": 5000
diff --git a/main/src/test/resources/parameters/MinimumParameters.json b/main/src/test/resources/parameters/MinimumParameters.json
index e611dc90..99c8a3a3 100644
--- a/main/src/test/resources/parameters/MinimumParameters.json
+++ b/main/src/test/resources/parameters/MinimumParameters.json
@@ -7,6 +7,7 @@
         "password":"zb!XztG34"
     },
     "pdpParameters": {
+        "heartBeatMs": 1,
         "updateParameters": {
             "maxRetryCount": 1,
             "maxWaitMs": 1
diff --git a/main/src/test/resources/parameters/PapConfigParameters.json b/main/src/test/resources/parameters/PapConfigParameters.json
index 2c24e4fe..16b22ae5 100644
--- a/main/src/test/resources/parameters/PapConfigParameters.json
+++ b/main/src/test/resources/parameters/PapConfigParameters.json
@@ -8,6 +8,7 @@
         "https": true
     },
     "pdpParameters": {
+        "heartBeatMs": 10,
         "updateParameters": {
             "maxRetryCount": 1,
             "maxWaitMs": 1
diff --git a/main/src/test/resources/parameters/PapConfigParametersStd.json b/main/src/test/resources/parameters/PapConfigParametersStd.json
index 6089c0b7..853249e0 100644
--- a/main/src/test/resources/parameters/PapConfigParametersStd.json
+++ b/main/src/test/resources/parameters/PapConfigParametersStd.json
@@ -15,7 +15,8 @@
         "stateChangeParameters": {
             "maxRetryCount": 1,
             "maxWaitMs": 5
-        }
+        },
+        "heartBeatMs": 6
     },
     "databaseProviderParameters": {
         "name": "PolicyModelsProviderParameters",
diff --git a/main/src/test/resources/parameters/PapConfigParameters_InvalidName.json b/main/src/test/resources/parameters/PapConfigParameters_InvalidName.json
index d06ecfc8..15e608ba 100644
--- a/main/src/test/resources/parameters/PapConfigParameters_InvalidName.json
+++ b/main/src/test/resources/parameters/PapConfigParameters_InvalidName.json
@@ -7,6 +7,7 @@
         "password":"zb!XztG34"
     },
     "pdpParameters": {
+        "heartBeatMs": 1,
         "updateParameters": {
             "maxRetryCount": 1,
             "maxWaitMs": 1
diff --git a/main/src/test/resources/parameters/PapConfigParameters_sim.json b/main/src/test/resources/parameters/PapConfigParameters_sim.json
index a9a68f90..96bbedd0 100644
--- a/main/src/test/resources/parameters/PapConfigParameters_sim.json
+++ b/main/src/test/resources/parameters/PapConfigParameters_sim.json
@@ -8,6 +8,7 @@
         "https": true
     },
     "pdpParameters": {
+        "heartBeatMs": 10,
         "updateParameters": {
             "maxRetryCount": 1,
             "maxWaitMs": 30000