Adding retry logic for SDC Client in Distribution

* SdcReceptionHandler in distribution tries to initialize & start the
SDC Client when distribution framework comes up. If by any chance the
SDC service is not up then currently the SdcReceptionHandler throws an
exception to the distribution activator and the distribution framework
goes down. And the health check fails.

* This review fixes it by adding a retry mechanism in
SdcReceptionHandler for all lifecycle methods of SdcClient (init, start,
stop). After failure the same operation is retried again with some
delay. The delay is passed as parameter from configuration json file.
The minimum default value for the delay is kept as 30 seconds.

* Adding SdcClientHandler timer task for performing the retries
asynchronously.

Change-Id: Ibb6d936fcf4872c82f87e2cd04a00583b81c92ff
Issue-ID: POLICY-1035
Signed-off-by: ramverma <ram.krishna.verma@ericsson.com>

4 files changed, 156 insertions, 44 deletions

diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java
new file mode 100644
index 00000000..f69fb155
--- /dev/null
+++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcClientHandler.java
@@ -0,0 +1,76 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2018 Ericsson. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.distribution.reception.handling.sdc;
+
+import java.util.Timer;
+import java.util.TimerTask;
+
+/**
+ * This class implements TimerTask for calling life cycle methods of SdcClient iteratively after specified interval
+ * until the operation is successful.
+ *
+ * @author Ram Krishna Verma (ram.krishna.verma@ericsson.com)
+ */
+public class SdcClientHandler extends TimerTask {
+
+    private SdcReceptionHandler sdcReceptionHandler;
+    private SdcClientOperationType operationType;
+    private Timer timer;
+
+    public enum SdcClientOperationType {
+        START, STOP
+    }
+
+    /**
+     * Constructs an instance of {@link SdcClientHandler} class.
+     *
+     * @param sdcReceptionHandler the sdcReceptionHandler
+     */
+    public SdcClientHandler(final SdcReceptionHandler sdcReceptionHandler, final SdcClientOperationType operationType,
+            final long retryDelay) {
+        this.sdcReceptionHandler = sdcReceptionHandler;
+        this.operationType = operationType;
+        timer = new Timer(false);
+        timer.scheduleAtFixedRate(this, 0, retryDelay * 1000L);
+    }
+
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public void run() {
+        if (SdcClientOperationType.START.equals(operationType)) {
+            sdcReceptionHandler.initializeSdcClient();
+            sdcReceptionHandler.startSdcClient();
+        } else {
+            sdcReceptionHandler.stopSdcClient();
+        }
+    }
+
+    /**
+     * {@inheritDoc}.
+     */
+    @Override
+    public boolean cancel() {
+        timer.cancel();
+        return true;
+    }
+}
diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java
index 443235b7..2ada9fcd 100644
--- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java
+++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandler.java
@@ -30,10 +30,9 @@ import org.onap.policy.common.logging.flexlogger.FlexLogger;
 import org.onap.policy.common.logging.flexlogger.Logger;
 import org.onap.policy.common.parameters.ParameterService;
 import org.onap.policy.distribution.model.Csar;
-import org.onap.policy.distribution.reception.decoding.PluginInitializationException;
-import org.onap.policy.distribution.reception.decoding.PluginTerminationException;
 import org.onap.policy.distribution.reception.decoding.PolicyDecodingException;
 import org.onap.policy.distribution.reception.handling.AbstractReceptionHandler;
+import org.onap.policy.distribution.reception.handling.sdc.SdcClientHandler.SdcClientOperationType;
 import org.onap.policy.distribution.reception.handling.sdc.exceptions.ArtifactDownloadException;
 import org.onap.policy.distribution.reception.statistics.DistributionStatisticsManager;
 import org.onap.sdc.api.IDistributionClient;
@@ -54,38 +53,34 @@ import org.onap.sdc.utils.DistributionStatusEnum;
 public class SdcReceptionHandler extends AbstractReceptionHandler implements INotificationCallback {
 
     private static final Logger LOGGER = FlexLogger.getLogger(SdcReceptionHandler.class);
+    private static final String SECONDS = "Seconds";
 
     private SdcReceptionHandlerStatus sdcReceptionHandlerStatus = SdcReceptionHandlerStatus.STOPPED;
-    private SdcReceptionHandlerConfigurationParameterGroup handlerParameters;
     private IDistributionClient distributionClient;
     private SdcConfiguration sdcConfig;
     private volatile int nbOfNotificationsOngoing = 0;
+    private int retryDelay;
+    private SdcClientHandler sdcClientHandler;
 
     private enum DistributionStatusType {
         DOWNLOAD, DEPLOY
     }
 
     @Override
-    protected void initializeReception(final String parameterGroupName) throws PluginInitializationException {
-        handlerParameters = ParameterService.get(parameterGroupName);
-        initializeSdcClient();
-        startSdcClient();
+    protected void initializeReception(final String parameterGroupName) {
+        final SdcReceptionHandlerConfigurationParameterGroup handlerParameters =
+                ParameterService.get(parameterGroupName);
+        retryDelay = handlerParameters.getRetryDelay() < 30 ? 30 : handlerParameters.getRetryDelay();
+        sdcConfig = new SdcConfiguration(handlerParameters);
+        distributionClient = createSdcDistributionClient();
+        sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.START, retryDelay);
     }
 
     @Override
-    public void destroy() throws PluginTerminationException {
-        LOGGER.debug("Going to stop the SDC Client...");
+    public void destroy() {
         if (distributionClient != null) {
-            final IDistributionClientResult clientResult = distributionClient.stop();
-            if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-                final String message =
-                        "SDC client stop failed with reason:" + clientResult.getDistributionMessageResult();
-                LOGGER.error(message);
-                throw new PluginTerminationException(message);
-            }
+            sdcClientHandler = new SdcClientHandler(this, SdcClientOperationType.STOP, retryDelay);
         }
-        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-        LOGGER.debug("SDC Client is stopped successfully");
     }
 
     @Override
@@ -115,6 +110,8 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo
                 ++nbOfNotificationsOngoing;
                 sdcReceptionHandlerStatus = newStatus;
                 break;
+            default:
+                break;
         }
     }
 
@@ -130,48 +127,61 @@ public class SdcReceptionHandler extends AbstractReceptionHandler implements INo
     /**
      * Method to initialize the SDC client.
      *
-     * @throws PluginInitializationException if the initialization of SDC Client fails
      */
-    private void initializeSdcClient() throws PluginInitializationException {
+    protected void initializeSdcClient() {
 
         LOGGER.debug("Initializing the SDC Client...");
         if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.STOPPED) {
-            final String message = "The SDC Client is already initialized";
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("The SDC Client is already initialized");
+            return;
         }
-        sdcConfig = new SdcConfiguration(handlerParameters);
-        distributionClient = createSdcDistributionClient();
         final IDistributionClientResult clientResult = distributionClient.init(sdcConfig, this);
         if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-            changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-            final String message =
-                    "SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult();
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("SDC client initialization failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Initialization will be retried after " + retryDelay + SECONDS);
+            return;
         }
         LOGGER.debug("SDC Client is initialized successfully");
-        this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT);
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.INIT);
     }
 
     /**
      * Method to start the SDC client.
      *
-     * @param configParameter the configuration parameters
-     * @throws PluginInitializationException if the start of SDC Client fails
      */
-    private void startSdcClient() throws PluginInitializationException {
+    protected void startSdcClient() {
 
         LOGGER.debug("Going to start the SDC Client...");
+        if (sdcReceptionHandlerStatus != SdcReceptionHandlerStatus.INIT) {
+            LOGGER.error("The SDC Client is not initialized");
+            return;
+        }
         final IDistributionClientResult clientResult = distributionClient.start();
         if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
-            changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
-            final String message = "SDC client start failed with reason:" + clientResult.getDistributionMessageResult();
-            LOGGER.error(message);
-            throw new PluginInitializationException(message);
+            LOGGER.error("SDC client start failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Start will be retried after " + retryDelay + SECONDS);
+            return;
         }
         LOGGER.debug("SDC Client is started successfully");
-        this.changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE);
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.IDLE);
+        sdcClientHandler.cancel();
+    }
+
+    /**
+     * Method to stop the SDC client.
+     *
+     */
+    protected void stopSdcClient() {
+        LOGGER.debug("Going to stop the SDC Client...");
+        final IDistributionClientResult clientResult = distributionClient.stop();
+        if (!clientResult.getDistributionActionResult().equals(DistributionActionResultEnum.SUCCESS)) {
+            LOGGER.error("SDC client stop failed with reason:" + clientResult.getDistributionMessageResult()
+                    + ". Stop will be retried after " + retryDelay + SECONDS);
+            return;
+        }
+        LOGGER.debug("SDC Client is stopped successfully");
+        changeSdcReceptionHandlerStatus(SdcReceptionHandlerStatus.STOPPED);
+        sdcClientHandler.cancel();
     }
 
     /**
diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java
index 34eaf33e..65305c1c 100644
--- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java
+++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterBuilder.java
@@ -44,6 +44,7 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
     private String keystorePassword;
     private List<String> messageBusAddress;
     private List<String> artifactTypes;
+    private int retryDelay;
 
     /**
      * Set activeserverTlsAuth to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
@@ -197,6 +198,16 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
     }
 
     /**
+     * Set retryDelay to this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
+     *
+     * @param retryDelay the retryDelay
+     */
+    public SdcReceptionHandlerConfigurationParameterBuilder setRetryDelay(final int retryDelay) {
+        this.retryDelay = retryDelay;
+        return this;
+    }
+
+    /**
      * Returns the active server TlsAuth of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
      *
      * @return the activeserverTlsAuth
@@ -332,6 +343,15 @@ public class SdcReceptionHandlerConfigurationParameterBuilder {
         return artifactTypes;
     }
 
+    /**
+     * Returns the retryDelay of this {@link SdcReceptionHandlerConfigurationParameterBuilder} instance.
+     *
+     * @return the retryDelay
+     */
+    public int getRetryDelay() {
+        return retryDelay;
+    }
+
 }
 
 
diff --git a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java
index 0140fb53..bdc9767d 100644
--- a/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java
+++ b/plugins/reception-plugins/src/main/java/org/onap/policy/distribution/reception/handling/sdc/SdcReceptionHandlerConfigurationParameterGroup.java
@@ -21,15 +21,15 @@
 package org.onap.policy.distribution.reception.handling.sdc;
 
 import java.util.List;
+
 import org.onap.policy.common.parameters.GroupValidationResult;
 import org.onap.policy.common.parameters.ValidationStatus;
 import org.onap.policy.common.utils.validation.ParameterValidationUtils;
 import org.onap.policy.distribution.reception.parameters.ReceptionHandlerConfigurationParameterGroup;
 
 /**
- * This class handles reading, parsing and validating of the Policy SDC Service Distribution
- * parameters from Json format, which strictly adheres to the interface:IConfiguration, defined by
- * SDC SDK.
+ * This class handles reading, parsing and validating of the Policy SDC Service Distribution parameters from Json
+ * format, which strictly adheres to the interface:IConfiguration, defined by SDC SDK.
  */
 public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHandlerConfigurationParameterGroup {
 
@@ -39,6 +39,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
     private String password;
     private int pollingInterval;
     private int pollingTimeout;
+    private int retryDelay;
     private String consumerId;
     private List<String> artifactTypes;
     private String consumerGroup;
@@ -50,8 +51,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
     private boolean isUseHttpsWithDmaap;
 
     /**
-     * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup}
-     * class.
+     * The constructor for instantiating {@link SdcReceptionHandlerConfigurationParameterGroup} class.
      *
      * @param builder the SDC configuration builder
      */
@@ -63,6 +63,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         password = builder.getPassword();
         pollingInterval = builder.getPollingInterval();
         pollingTimeout = builder.getPollingTimeout();
+        retryDelay = builder.getRetryDelay();
         consumerId = builder.getConsumerId();
         artifactTypes = builder.getArtifactTypes();
         consumerGroup = builder.getConsumerGroup();
@@ -99,6 +100,10 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         return pollingTimeout;
     }
 
+    public int getRetryDelay() {
+        return retryDelay;
+    }
+
     public String getConsumerId() {
         return consumerId;
     }
@@ -149,6 +154,7 @@ public class SdcReceptionHandlerConfigurationParameterGroup extends ReceptionHan
         validateStringElement(validationResult, keyStorePassword, "keyStorePassword");
         validateIntElement(validationResult, pollingInterval, "pollingInterval");
         validateIntElement(validationResult, pollingTimeout, "pollingTimeout");
+        validateIntElement(validationResult, retryDelay, "retryDelay");
         validateStringListElement(validationResult, messageBusAddress, "messageBusAddress");
         validateStringListElement(validationResult, artifactTypes, "artifactTypes");
         return validationResult;