Adding retry logic for SDC Client in Distribution

* SdcReceptionHandler in distribution tries to initialize & start the
SDC Client when distribution framework comes up. If by any chance the
SDC service is not up then currently the SdcReceptionHandler throws an
exception to the distribution activator and the distribution framework
goes down. And the health check fails.

* This review fixes it by adding a retry mechanism in
SdcReceptionHandler for all lifecycle methods of SdcClient (init, start,
stop). After failure the same operation is retried again with some
delay. The delay is passed as parameter from configuration json file.
The minimum default value for the delay is kept as 30 seconds.

* Adding SdcClientHandler timer task for performing the retries
asynchronously.

Change-Id: Ibb6d936fcf4872c82f87e2cd04a00583b81c92ff
Issue-ID: POLICY-1035
Signed-off-by: ramverma <ram.krishna.verma@ericsson.com>

3 files changed, 31 insertions, 36 deletions

diff --git a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java
index bf42476f..557a7683 100644
--- a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java
+++ b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandler.java
@@ -51,7 +51,6 @@ import org.onap.policy.distribution.forwarding.PolicyForwarder;
 import org.onap.policy.distribution.forwarding.parameters.PolicyForwarderParameters;
 import org.onap.policy.distribution.model.Csar;
 import org.onap.policy.distribution.reception.decoding.PluginInitializationException;
-import org.onap.policy.distribution.reception.decoding.PluginTerminationException;
 import org.onap.policy.distribution.reception.decoding.PolicyDecoder;
 import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler;
 import org.onap.policy.distribution.reception.handling.PluginHandler;
@@ -97,9 +96,14 @@ public class TestSdcReceptionHandler {
      * Setup for the test cases.
      *
      * @throws IOException if it occurs
+     * @throws SecurityException if it occurs
+     * @throws NoSuchFieldException if it occurs
+     * @throws IllegalAccessException if it occurs
+     * @throws IllegalArgumentException if it occurs
      */
     @Before
-    public final void init() throws IOException {
+    public final void init() throws IOException, NoSuchFieldException, SecurityException, IllegalArgumentException,
+            IllegalAccessException {
         DistributionStatisticsManager.resetAllStatistics();
         final Gson gson = new GsonBuilder().create();
         pssdConfigParameters = gson.fromJson(new FileReader("src/test/resources/handling-sdc.json"),
@@ -140,45 +144,34 @@ public class TestSdcReceptionHandler {
     public final void testInitializeSdcClient() {
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
-        } catch (final PluginInitializationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
     }
 
     @Test
-    public final void testInitializeSdcClient_Again() throws PluginInitializationException {
-        sypHandler.initializeReception(pssdConfigParameters.getName());
-        try {
-            sypHandler.initializeReception(pssdConfigParameters.getName());
-            fail("Test must throw an exception here");
-        } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("The SDC Client is already initialized"));
-        }
-    }
-
-    @Test
-    public final void testInitializeSdcClient_Failure() throws PluginInitializationException {
+    public final void testInitializeSdcClient_Failure() {
 
         Mockito.when(successfulClientInitResult.getDistributionActionResult())
-                .thenReturn(DistributionActionResultEnum.FAIL);
+                .thenReturn(DistributionActionResultEnum.FAIL).thenReturn(DistributionActionResultEnum.SUCCESS);
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client initialization failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
     }
 
     @Test
-    public final void testStartSdcClient_Failure() throws PluginInitializationException {
+    public final void testStartSdcClient_Failure() {
         try {
-            Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult);
+            Mockito.when(distributionClient.start()).thenReturn(failureClientInitResult)
+                    .thenReturn(successfulClientInitResult);
             sypHandler.initializeReception(pssdConfigParameters.getName());
-
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client start failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
     }
 
@@ -187,7 +180,7 @@ public class TestSdcReceptionHandler {
         try {
             sypHandler.initializeReception(pssdConfigParameters.getName());
             sypHandler.destroy();
-        } catch (final PluginInitializationException | PluginTerminationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
@@ -195,28 +188,28 @@ public class TestSdcReceptionHandler {
     }
 
     @Test
-    public final void testStopSdcClientWithoutStart() {
+    public final void testStopSdcClient_Failure() throws PluginInitializationException {
+
+        sypHandler.initializeReception(pssdConfigParameters.getName());
+        Mockito.when(distributionClient.stop()).thenReturn(failureClientInitResult)
+                .thenReturn(successfulClientInitResult);
         try {
             sypHandler.destroy();
-        } catch (final PluginTerminationException exp) {
+        } catch (final Exception exp) {
             LOGGER.error(exp);
             fail("Test should not throw any exception");
         }
-
     }
 
     @Test
-    public final void testStopSdcClient_Failure() throws PluginInitializationException {
-
-        sypHandler.initializeReception(pssdConfigParameters.getName());
-        Mockito.when(successfulClientInitResult.getDistributionActionResult())
-                .thenReturn(DistributionActionResultEnum.FAIL);
+    public final void testStopSdcClientWithoutStart() {
         try {
             sypHandler.destroy();
-            fail("Test must throw an exception here");
         } catch (final Exception exp) {
-            assertTrue(exp.getMessage().startsWith("SDC client stop failed with reason"));
+            LOGGER.error(exp);
+            fail("Test should not throw any exception");
         }
+
     }
 
     @Test
diff --git a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java
index df72d45c..f9a65c54 100644
--- a/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java
+++ b/plugins/reception-plugins/src/test/java/org/onap/policy/distribution/reception/handling/sdc/TestSdcReceptionHandlerConfigurationParameterGroup.java
@@ -92,8 +92,8 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup {
                 new SdcReceptionHandlerConfigurationParameterBuilder().setAsdcAddress("localhost")
                         .setConsumerGroup("policy-group").setConsumerId("policy-id").setEnvironmentName("TEST")
                         .setKeystorePassword("password").setKeystorePath("dummyPath").setPassword("policy")
-                        .setPollingInterval(10).setPollingTimeout(20).setUser("policy").setUseHttpsWithDmaap(false)
-                        .setActiveserverTlsAuth(false).setFilterinEmptyResources(true)
+                        .setPollingInterval(10).setPollingTimeout(20).setRetryDelay(30).setUser("policy")
+                        .setUseHttpsWithDmaap(false).setActiveserverTlsAuth(false).setFilterinEmptyResources(true)
                         .setArtifactTypes(Arrays.asList("TOSCA_CSAR")).setMessageBusAddress(Arrays.asList("localhost"));
         final SdcReceptionHandlerConfigurationParameterGroup configParameters =
                 new SdcReceptionHandlerConfigurationParameterGroup(builder);
@@ -106,6 +106,7 @@ public class TestSdcReceptionHandlerConfigurationParameterGroup {
         assertEquals("policy", configParameters.getPassword());
         assertEquals(10, configParameters.getPollingInterval());
         assertEquals(20, configParameters.getPollingTimeout());
+        assertEquals(30, configParameters.getRetryDelay());
         assertEquals("policy-id", configParameters.getConsumerId());
         assertEquals("policy-group", configParameters.getConsumerGroup());
         assertEquals("TEST", configParameters.getEnvironmentName());
diff --git a/plugins/reception-plugins/src/test/resources/handling-sdc.json b/plugins/reception-plugins/src/test/resources/handling-sdc.json
index 7e3758de..0bb936bf 100644
--- a/plugins/reception-plugins/src/test/resources/handling-sdc.json
+++ b/plugins/reception-plugins/src/test/resources/handling-sdc.json
@@ -10,6 +10,7 @@
     "password": "policy",
     "pollingInterval":20,
     "pollingTimeout":30,
+    "retryDelay":30,
     "consumerId": "policy-id",
     "artifactTypes": [
         "TOSCA_CSAR",