[APPC-144] OAM operation abort messages

Provide abort audit log as well as notification when OAM operation is
interrupted by a new OAM operation request.
Fix missing ID in audit log for reject message.

Issue-Id: APPC-144
Change-Id: Ie87e19949be85c085444c753fdf061f4fc45e48f
Signed-off-by: beili.zhou <beili.zhou@amdocs.com>

10 files changed, 289 insertions, 53 deletions

diff --git a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/OAMCommandStatus.java b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/OAMCommandStatus.java
index 378fa032d..b8a4e3e86 100644
--- a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/OAMCommandStatus.java
+++ b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/OAMCommandStatus.java
@@ -24,7 +24,6 @@
 
 package org.openecomp.appc.oam;
 
-
 import org.openecomp.appc.executor.objects.Params;
 import org.openecomp.appc.util.MessageFormatter;
 
@@ -32,24 +31,19 @@ import java.util.Map;
 
 public enum OAMCommandStatus {
 
-    ACCEPTED(100,"ACCEPTED - request accepted"),
-
-    //ERROR(2xx) - request can't be handled due to some technical error
-    UNEXPECTED_ERROR(200,"UNEXPECTED ERROR - ${errorMsg}"),
-
-    REJECTED(300,"REJECTED - ${errorMsg}"),
-    INVALID_PARAMETER(302,"INVALID PARAMETER - ${errorMsg}" ),
-    TIMEOUT(303, "OPERATION TIMEOUT REACHED - ${errorMsg}"),
+    ACCEPTED(100,          "ACCEPTED - request accepted"),
+    UNEXPECTED_ERROR(200,  "UNEXPECTED ERROR - ${errorMsg}"),
+    REJECTED(300,          "REJECTED - ${errorMsg}"),
+    INVALID_PARAMETER(302, "INVALID PARAMETER - ${errorMsg}" ),
+    TIMEOUT(303,           "OPERATION TIMEOUT REACHED - ${errorMsg}"),
+    ABORT(304,             "OPERATION ABORT - ${errorMsg}"),
+    SUCCESS(400,           "SUCCESS - request has been processed successfully");
 
-    SUCCESS(400,"SUCCESS - request has been processed successfully"),
-    ;
+    final String TO_STRING_FORMAT = "OAMCommandStatus{responseCode=%d, responseMessage='%s'}";
 
     private int responseCode;
     private String responseMessage;
 
-
-
-
     OAMCommandStatus(int responseCode, String responseMessage) {
         this.responseCode = responseCode;
         this.responseMessage = responseMessage;
@@ -63,22 +57,18 @@ public enum OAMCommandStatus {
         return responseCode;
     }
 
-
     /**
+     * Get formated message of passed in params
      *
-     * @return  messageTemplate
+     * @param params of Params object with name value pairs for message
+     * @return  message string
      */
-    public String getFormattedMessage(Params params){
+    public String getFormattedMessage(Params params) {
         Map<String,Object> paramsMap = params != null ? params.getParams() : null;
-        return MessageFormatter.format(getResponseMessage(),paramsMap);
-
+        return MessageFormatter.format(getResponseMessage(), paramsMap);
     }
 
     @Override
     public String toString() {
-        return "OAMCommandStatus{" +
-                "responseCode=" + responseCode +
-                ", responseMessage='" + responseMessage + '\'' +
-                '}';
-    }
-}
+        return String.format(TO_STRING_FORMAT, responseCode, responseMessage);
+    }}
diff --git a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseActionRunnable.java b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseActionRunnable.java
index 2ffad6993..542e53d63 100644
--- a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseActionRunnable.java
+++ b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseActionRunnable.java
@@ -25,6 +25,7 @@
 package org.openecomp.appc.oam.processor;
 
 import org.openecomp.appc.i18n.Msg;
+import org.openecomp.appc.oam.AppcOam;
 import org.openecomp.appc.oam.OAMCommandStatus;
 import org.openecomp.appc.statemachine.impl.readers.AppcOamStates;
 
@@ -43,16 +44,18 @@ import java.util.concurrent.Future;
  *   <br>  - auditMsg
  *   <br>  - finalState
  */
-abstract class BaseActionRunnable extends BaseCommon implements Runnable {
+public abstract class BaseActionRunnable extends BaseCommon implements Runnable {
     final String OAM_OPERATION_TIMEOUT_SECOND = "appc.OAM.api.timeout";
     /** Default operation tiemout set to 1 minute */
     final int DEFAULT_OAM_OPERATION_TIMEOUT = 60;
-    /** Abort message format with flexible operation name */
-    final String ABORT_MESSAGE_FORMAT = "Aborting %s operation.";
+    /** Abort due to rejection message format with flexible operation name */
+    final String ABORT_MESSAGE_FORMAT = "Aborting %s operation due to %s.";
     /** Timeout message format with flexible operation name */
     final String TIMEOUT_MESSAGE_FORMAT = "%s operation has reached timeout %d milliseconds.";
     /** Failure message format with flexible number of bundles */
     final String BUNDLE_OPERATION_FAILED_FORMAT = "%d bundle(s) failed, see logs for details.";
+    final String NEW_RPC_OPERATION_REQUEST = "new %s operation request";
+    final String DUE_TO_EXECUTION_ERROR = "due to execution error.";
 
     private boolean isWaiting = false;
     private AppcOamStates currentState;
@@ -103,6 +106,26 @@ abstract class BaseActionRunnable extends BaseCommon implements Runnable {
                 rpc.name(), Boolean.toString(doTimeoutChecking), timeoutMs, startTimeMs);
     }
 
+    /**
+     * Abort operation handling due to outside interruption, does<br>
+     *     - set ABORT status<br>
+     *     - send notification message<br>
+     *     - add audit log
+     *
+     * @param newRpc of the new AppcOam.RPC operation.
+     */
+    public void abortRunnable(final AppcOam.RPC newRpc) {
+        resetLogProperties(false);
+
+        String additionalMsg = String.format(NEW_RPC_OPERATION_REQUEST, newRpc);
+        logDebug("%s action aborted due to %s", rpc.name(), additionalMsg);
+        setStatus(OAMCommandStatus.ABORT, String.format(ABORT_MESSAGE_FORMAT, rpc.name(), additionalMsg));
+        operationHelper.sendNotificationMessage(rpc, commonHeader, status);
+        auditInfoLog(auditMsg);
+
+        resetLogProperties(true);
+    }
+
     @Override
     public void run() {
         try {
@@ -179,10 +202,10 @@ abstract class BaseActionRunnable extends BaseCommon implements Runnable {
     }
 
     /**
-     * Set class <b>status</b> to REJECTED with abort message.
+     * Set class <b>status</b> to ABORT with abort message.
      */
     void setAbortStatus() {
-        setStatus(OAMCommandStatus.REJECTED, String.format(ABORT_MESSAGE_FORMAT, rpc.name()));
+        setStatus(OAMCommandStatus.ABORT, String.format(ABORT_MESSAGE_FORMAT, rpc.name(), DUE_TO_EXECUTION_ERROR));
     }
 
     /**
diff --git a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseCommon.java b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseCommon.java
index cc725e7b9..ccb57305a 100644
--- a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseCommon.java
+++ b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseCommon.java
@@ -43,7 +43,11 @@ import org.openecomp.appc.oam.util.StateHelper;
 import org.slf4j.MDC;
 
 import java.net.InetAddress;
+import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import static com.att.eelf.configuration.Configuration.MDC_INSTANCE_UUID;
 import static com.att.eelf.configuration.Configuration.MDC_KEY_REQUEST_ID;
@@ -69,6 +73,18 @@ abstract class BaseCommon {
     AppcOam.RPC rpc;
     CommonHeader commonHeader;
 
+    private final List<String> MDC_KEYS = Arrays.asList(
+       LoggingConstants.MDCKeys.PARTNER_NAME,
+       LoggingConstants.MDCKeys.SERVER_NAME,
+       MDC_INSTANCE_UUID,
+       MDC_KEY_REQUEST_ID,
+       MDC_SERVER_FQDN,
+       MDC_SERVER_IP_ADDRESS,
+       MDC_SERVICE_NAME
+    );
+
+    private Map<String, String> oldMdcContent = new HashMap<>();
+
     /**
      * Constructor
      *
@@ -138,6 +154,32 @@ abstract class BaseCommon {
     }
 
     /**
+     * Reset MDC log properties based on passed in condition. does:<br>
+     *   - persist existing MDC setting and set my MDC log properties <br>
+     *   - or re-apply persisted MDC log properties
+     * @param useMdcMap boolean to indicate whether to persist the existing MDC setting and set my MDC log properties,
+     *                  or to re-apply the persisted MDC log properties.
+     */
+    void resetLogProperties(boolean useMdcMap) {
+        if (useMdcMap) {
+            for (Map.Entry<String, String> aEntry : oldMdcContent.entrySet()) {
+                MDC.put(aEntry.getKey(), aEntry.getValue());
+            }
+            return;
+        }
+
+        // persist existing log properties and set my log properties
+        oldMdcContent.clear();
+        for (String key : MDC_KEYS) {
+            String value = MDC.get(key);
+            if (value != null) {
+                oldMdcContent.put(key, value);
+            }
+        }
+        setInitialLogProperties();
+    }
+
+    /**
      * Set class <b>status</b> by calling setStatus(OAMCommandStatus, Params) with null paramter.
      * @see #setStatus(OAMCommandStatus, String)
      *
@@ -170,9 +212,11 @@ abstract class BaseCommon {
     /**
      * Set class <b>status</b> with error status calculated from the passed in paremeter
      * and audit log the error message.
-     * @param t of the erro Throwable.
+     * @param t of the error Throwable.
      */
     void setErrorStatus(Throwable t) {
+        resetLogProperties(false);
+
         final String appName = configurationHelper.getAppcName();
         String exceptionMessage = t.getMessage() != null ? t.getMessage() : t.toString();
 
@@ -201,6 +245,8 @@ abstract class BaseCommon {
                 LoggingConstants.TargetNames.APPC_OAM_PROVIDER,
                 errorMessage,
                 AppcOam.class.getCanonicalName());
+
+        resetLogProperties(true);
     }
 
     /**
diff --git a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseProcessor.java b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseProcessor.java
index 6c7011111..c9ba0efa7 100644
--- a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseProcessor.java
+++ b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/processor/BaseProcessor.java
@@ -54,7 +54,7 @@ public abstract class BaseProcessor extends BaseCommon {
 
     Integer timeoutSeconds;
     Msg auditMsg;
-    Runnable runnable;
+    BaseActionRunnable runnable;
     private Future<?> scheduledRunnable = null;
 
     /**
diff --git a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/util/AsyncTaskHelper.java b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/util/AsyncTaskHelper.java
index ff28e995a..25467da04 100644
--- a/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/util/AsyncTaskHelper.java
+++ b/appc-oam/appc-oam-bundle/src/main/java/org/openecomp/appc/oam/util/AsyncTaskHelper.java
@@ -26,6 +26,7 @@ package org.openecomp.appc.oam.util;
 
 import com.att.eelf.configuration.EELFLogger;
 import org.openecomp.appc.oam.AppcOam;
+import org.openecomp.appc.oam.processor.BaseActionRunnable;
 import org.osgi.framework.Bundle;
 import org.osgi.framework.FrameworkUtil;
 
@@ -52,6 +53,8 @@ public class AsyncTaskHelper {
 
     /** Reference to the Async task */
     private volatile Future<?> backgroundOamTask;
+    /** Reference to the runnable of Async task */
+    private volatile BaseActionRunnable taskRunnable;
 
     /**
      * Constructor
@@ -107,9 +110,9 @@ public class AsyncTaskHelper {
      * Schedule a service for async task with the passed in parameters
      * @param rpc of the REST API call, decides how to schedule the service
      * @param runnable of the to be scheduled service.
-     * @return the refernce of the scheduled task
+     * @return the reference of the scheduled task
      */
-    public Future<?> scheduleAsyncTask(final AppcOam.RPC rpc, final Runnable runnable) {
+    public Future<?> scheduleAsyncTask(final AppcOam.RPC rpc, final BaseActionRunnable runnable) {
         int initialDelay, interval;
         switch (rpc) {
             case maintenance_mode:
@@ -129,8 +132,14 @@ public class AsyncTaskHelper {
 
         // Always cancel existing  async task
         if (backgroundOamTask != null) {
+            logDebug("Cancelling background task in schedule task.");
             backgroundOamTask.cancel(true);
+            if (taskRunnable != null) {
+                taskRunnable.abortRunnable(rpc);
+            }
         }
+
+        taskRunnable = runnable;
         backgroundOamTask = scheduledExecutorService.scheduleWithFixedDelay(
                 runnable, initialDelay, interval, TimeUnit.MILLISECONDS);
 
@@ -149,6 +158,7 @@ public class AsyncTaskHelper {
         task.cancel(false);
         if (task == backgroundOamTask) {
             backgroundOamTask = null;
+            logDebug("Cancelling background task in cancel task.");
         }
     }
 
diff --git a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/OAMCommandStatusTest.java b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/OAMCommandStatusTest.java
new file mode 100644
index 000000000..c941f43dc
--- /dev/null
+++ b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/OAMCommandStatusTest.java
@@ -0,0 +1,99 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP : APPC
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Copyright (C) 2017 Amdocs
+ * =============================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * 
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.appc.oam;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.openecomp.appc.executor.objects.Params;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class OAMCommandStatusTest {
+    private Map<OAMCommandStatus, Integer> CODE_MAP = new HashMap<OAMCommandStatus, Integer>() {
+        {
+            put(OAMCommandStatus.ABORT,             304);
+            put(OAMCommandStatus.ACCEPTED,          100);
+            put(OAMCommandStatus.INVALID_PARAMETER, 302);
+            put(OAMCommandStatus.REJECTED,          300);
+            put(OAMCommandStatus.SUCCESS,           400);
+            put(OAMCommandStatus.TIMEOUT,           303);
+            put(OAMCommandStatus.UNEXPECTED_ERROR,  200);
+        }
+    };
+    private Map<OAMCommandStatus, String> MSG_MAP = new HashMap<OAMCommandStatus, String>() {
+        {
+            put(OAMCommandStatus.ABORT,             "OPERATION ABORT - ${errorMsg}");
+            put(OAMCommandStatus.ACCEPTED,          "ACCEPTED - request accepted");
+            put(OAMCommandStatus.INVALID_PARAMETER, "INVALID PARAMETER - ${errorMsg}");
+            put(OAMCommandStatus.REJECTED,          "REJECTED - ${errorMsg}");
+            put(OAMCommandStatus.SUCCESS,           "SUCCESS - request has been processed successfully");
+            put(OAMCommandStatus.TIMEOUT,           "OPERATION TIMEOUT REACHED - ${errorMsg}");
+            put(OAMCommandStatus.UNEXPECTED_ERROR,  "UNEXPECTED ERROR - ${errorMsg}");
+        }
+    };
+
+    @Test
+    public void testGetResponseMessage() throws Exception {
+        for (OAMCommandStatus oamCommandStatus : OAMCommandStatus.values()) {
+            String expectedMsg = MSG_MAP.get(oamCommandStatus);
+            Assert.assertEquals(String.format("Should have message (%s).", expectedMsg),
+                    expectedMsg, oamCommandStatus.getResponseMessage());
+        }
+    }
+
+    @Test
+    public void testGetResponseCode() throws Exception {
+        for (OAMCommandStatus oamCommandStatus : OAMCommandStatus.values()) {
+            Integer expectedCode = CODE_MAP.get(oamCommandStatus);
+            Assert.assertEquals(String.format("Should have code (%d).", expectedCode),
+                    expectedCode, Integer.valueOf(oamCommandStatus.getResponseCode()));
+        }
+    }
+
+    @Test
+    public void testGetFormattedMessage() throws Exception {
+        String message = "test message";
+        Params params = new Params().addParam("errorMsg", message);
+        for (OAMCommandStatus oamCommandStatus : OAMCommandStatus.values()) {
+            String expectedMsg1 = MSG_MAP.get(oamCommandStatus);
+            String expectedMsg2 = expectedMsg1.replaceAll("\\$\\{errorMsg\\}", message);
+            Assert.assertEquals("Should returned " + expectedMsg1,
+                    expectedMsg1, oamCommandStatus.getFormattedMessage(null));
+            Assert.assertEquals("Should returned " + expectedMsg2,
+                    expectedMsg2, oamCommandStatus.getFormattedMessage(params));
+        }
+    }
+
+    @Test
+    public void testToString() throws Exception {
+        for (OAMCommandStatus oamCommandStatus : OAMCommandStatus.values()) {
+            String expectedString = String.format(oamCommandStatus.TO_STRING_FORMAT,
+                    CODE_MAP.get(oamCommandStatus), MSG_MAP.get(oamCommandStatus));
+            Assert.assertEquals(String.format("Should have string (%s).", expectedString),
+                    expectedString, oamCommandStatus.toString());
+        }
+    }
+}
diff --git a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseActionRunnableTest.java b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseActionRunnableTest.java
index 70adca0fc..c5ad95e43 100644
--- a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseActionRunnableTest.java
+++ b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseActionRunnableTest.java
@@ -186,11 +186,12 @@ public class BaseActionRunnableTest {
     @Test
     public void testSetAbortStatus() throws Exception {
         testBaseAcionRunnable.setAbortStatus();
-        Assert.assertEquals("Should return reject code", OAMCommandStatus.REJECTED.getResponseCode(),
+        Assert.assertEquals("Should return abort code", OAMCommandStatus.ABORT.getResponseCode(),
                 testBaseAcionRunnable.status.getCode().intValue());
-        Assert.assertTrue("Should set abort message",
+        Assert.assertTrue("Should set abort due to execution error message",
                 testBaseAcionRunnable.status.getMessage().endsWith(
-                        String.format(testBaseAcionRunnable.ABORT_MESSAGE_FORMAT, testRpc.name())));
+                        String.format(testBaseAcionRunnable.ABORT_MESSAGE_FORMAT,
+                                testRpc.name(), testBaseAcionRunnable.DUE_TO_EXECUTION_ERROR)));
     }
 
     @Test
@@ -281,4 +282,20 @@ public class BaseActionRunnableTest {
                 String.format(testBaseAcionRunnable.BUNDLE_OPERATION_FAILED_FORMAT, failedNumber));
         Mockito.verify(testBaseAcionRunnable, times(1)).postAction(AppcOamStates.Error);
     }
+
+    @Test
+    public void testAbortRunnable() throws Exception {
+        Mockito.doReturn(AppcOamStates.Restarting).when(mockStateHelper).getCurrentOamState();
+        AppcOam.RPC newRpc = AppcOam.RPC.restart;
+        testBaseAcionRunnable.abortRunnable(newRpc);
+        Assert.assertEquals("Should return abort code", OAMCommandStatus.ABORT.getResponseCode(),
+                testBaseAcionRunnable.status.getCode().intValue());
+        Assert.assertTrue("Should set abort due to new request message",
+                testBaseAcionRunnable.status.getMessage().endsWith(
+                        String.format(testBaseAcionRunnable.ABORT_MESSAGE_FORMAT, testRpc.name(),
+                                String.format(testBaseAcionRunnable.NEW_RPC_OPERATION_REQUEST, newRpc.name()))));
+        Mockito.verify(mockOperHelper, times(1)).sendNotificationMessage(any(), any(), any());
+        Mockito.verify(testBaseAcionRunnable, times(1)).resetLogProperties(false);
+        Mockito.verify(testBaseAcionRunnable, times(1)).resetLogProperties(true);
+    }
 }
diff --git a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseCommonTest.java b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseCommonTest.java
index 4e1025781..3a9e76f27 100644
--- a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseCommonTest.java
+++ b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseCommonTest.java
@@ -22,14 +22,15 @@
  * ============LICENSE_END=========================================================
  */
 
-
 package org.openecomp.appc.oam.processor;
 
 import com.att.eelf.configuration.EELFLogger;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 import org.mockito.Mockito;
+import org.opendaylight.yang.gen.v1.org.openecomp.appc.oam.rev170303.common.header.CommonHeader;
 import org.opendaylight.yang.gen.v1.org.openecomp.appc.oam.rev170303.status.Status;
 import org.openecomp.appc.exceptions.InvalidInputException;
 import org.openecomp.appc.exceptions.InvalidStateException;
@@ -39,10 +40,21 @@ import org.openecomp.appc.oam.util.ConfigurationHelper;
 import org.openecomp.appc.oam.util.OperationHelper;
 import org.openecomp.appc.oam.util.StateHelper;
 import org.openecomp.appc.statemachine.impl.readers.AppcOamStates;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
 import org.powermock.reflect.Whitebox;
+import org.slf4j.MDC;
+
+import java.util.Map;
 
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.powermock.api.mockito.PowerMockito.mockStatic;
 
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({BaseCommon.class, MDC.class})
 public class BaseCommonTest {
     private class TestAbc extends BaseCommon {
 
@@ -65,10 +77,14 @@ public class BaseCommonTest {
     private TestAbc testBaseCommon;
     private ConfigurationHelper mockConfigHelper = mock(ConfigurationHelper.class);
     private StateHelper mockStateHelper = mock(StateHelper.class);
+    private CommonHeader mockCommonHeader = mock(CommonHeader.class);
 
     @Before
     public void setUp() throws Exception {
-        testBaseCommon = new TestAbc(null, mockConfigHelper, mockStateHelper, null);
+        testBaseCommon = spy(new TestAbc(null, mockConfigHelper, mockStateHelper, null));
+
+        Whitebox.setInternalState(testBaseCommon, "commonHeader", mockCommonHeader);
+        Whitebox.setInternalState(testBaseCommon, "rpc", AppcOam.RPC.maintenance_mode);
 
         // to avoid operation on logger fail, mock up the logger
         EELFLogger mockLogger = mock(EELFLogger.class);
@@ -98,7 +114,8 @@ public class BaseCommonTest {
     public void testSetErrorStatus() throws Exception {
         Mockito.doReturn("testName").when(mockConfigHelper).getAppcName();
         Mockito.doReturn(AppcOamStates.Started).when(mockStateHelper).getCurrentOamState();
-        Whitebox.setInternalState(testBaseCommon, "rpc", AppcOam.RPC.maintenance_mode);
+        Mockito.doReturn("testRequestId").when(mockCommonHeader).getRequestId();
+        Mockito.doReturn("testOrigId").when(mockCommonHeader).getOriginatorId();
 
         String exceptionMessage = "testing";
 
@@ -107,18 +124,50 @@ public class BaseCommonTest {
         testBaseCommon.setErrorStatus(t);
         Status status = testBaseCommon.status;
         Assert.assertEquals("Should have code", oamCommandStatus.getResponseCode(), status.getCode().intValue());
+        Mockito.verify(testBaseCommon, times(1)).resetLogProperties(false);
+        Mockito.verify(testBaseCommon, times(1)).resetLogProperties(true);
 
         oamCommandStatus = OAMCommandStatus.REJECTED;
         t = new InvalidStateException(exceptionMessage);
         testBaseCommon.setErrorStatus(t);
         status = testBaseCommon.status;
         Assert.assertEquals("Should have code", oamCommandStatus.getResponseCode(), status.getCode().intValue());
+        Mockito.verify(testBaseCommon, times(2)).resetLogProperties(false);
+        Mockito.verify(testBaseCommon, times(2)).resetLogProperties(true);
 
         oamCommandStatus = OAMCommandStatus.UNEXPECTED_ERROR;
         t = new NullPointerException(exceptionMessage);
         testBaseCommon.setErrorStatus(t);
         status = testBaseCommon.status;
         Assert.assertEquals("Should have code", oamCommandStatus.getResponseCode(), status.getCode().intValue());
+        Mockito.verify(testBaseCommon, times(3)).resetLogProperties(false);
+        Mockito.verify(testBaseCommon, times(3)).resetLogProperties(true);
+    }
+
+    @Test
+    public void testSetInitialLogProperties() throws Exception {
+        mockStatic(MDC.class);
+        testBaseCommon.setInitialLogProperties();
+        PowerMockito.verifyStatic(times(5));
     }
 
+    @Test
+    public void testClearRequestLogProperties() throws Exception {
+        mockStatic(MDC.class);
+        testBaseCommon.clearRequestLogProperties();
+        PowerMockito.verifyStatic(times(5));
+    }
+
+    @Test
+    public void testResetLogProperties() throws Exception {
+        testBaseCommon.setInitialLogProperties();
+
+        testBaseCommon.resetLogProperties(false);
+        Mockito.verify(testBaseCommon, times(2)).setInitialLogProperties();
+        Map<String, String> oldMdcMap = Whitebox.getInternalState(testBaseCommon, "oldMdcContent");
+        Assert.assertTrue("Should have 5 entries in persisted map", oldMdcMap.size() == 5);
+
+        testBaseCommon.resetLogProperties(false);
+        Mockito.verify(testBaseCommon, times(3)).setInitialLogProperties();
+    }
 }
diff --git a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseProcessorTest.java b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseProcessorTest.java
index 7a023e162..354053c8e 100644
--- a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseProcessorTest.java
+++ b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/processor/BaseProcessorTest.java
@@ -165,13 +165,10 @@ public class BaseProcessorTest {
         testBaseProcessor.scheduleAsyncTask();
         Mockito.verify(mockTaskHelper, times(0)).scheduleAsyncTask(any(), any());
 
-        // test runnable
-        Runnable runnable = () -> {
-            // do nothing
-        };
-        Whitebox.setInternalState(testBaseProcessor, "runnable", runnable);
+        BaseActionRunnable mockRunnable = mock(BaseActionRunnable.class);
+        Whitebox.setInternalState(testBaseProcessor, "runnable", mockRunnable);
         testBaseProcessor.scheduleAsyncTask();
-        Mockito.verify(mockTaskHelper, times(1)).scheduleAsyncTask(testRpc, runnable);
+        Mockito.verify(mockTaskHelper, times(1)).scheduleAsyncTask(testRpc, mockRunnable);
     }
 
     @Test
diff --git a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/util/AsyncTaskHelperTest.java b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/util/AsyncTaskHelperTest.java
index d080b735b..873b21795 100644
--- a/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/util/AsyncTaskHelperTest.java
+++ b/appc-oam/appc-oam-bundle/src/test/java/org/openecomp/appc/oam/util/AsyncTaskHelperTest.java
@@ -30,6 +30,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 import org.openecomp.appc.oam.AppcOam;
+import org.openecomp.appc.oam.processor.BaseActionRunnable;
 import org.powermock.reflect.Whitebox;
 
 import java.util.Arrays;
@@ -44,6 +45,7 @@ import static org.mockito.Mockito.times;
 public class AsyncTaskHelperTest {
     private AsyncTaskHelper asyncTaskHelper;
     private ScheduledExecutorService mockScheduler = mock(ScheduledExecutorService.class);
+    private BaseActionRunnable mockRunnable = mock(BaseActionRunnable.class);
 
     @Before
     public void setUp() throws Exception {
@@ -70,15 +72,14 @@ public class AsyncTaskHelperTest {
 
     @Test
     public void testScheduleAsyncTaskWithMmod() throws Exception {
-        Runnable mockRunnable = mock(Runnable.class);
-
-        // test maintance mode
+        // test maintenance mode
         ScheduledFuture<?> mockTask0 = mock(ScheduledFuture.class);
         Whitebox.setInternalState(asyncTaskHelper, "backgroundOamTask", mockTask0);
 
         ScheduledFuture<?> mockTask1 = mock(ScheduledFuture.class);
         Mockito.doReturn(mockTask1).when(mockScheduler).scheduleWithFixedDelay(
-                mockRunnable, asyncTaskHelper.MMODE_TASK_DELAY, asyncTaskHelper.MMODE_TASK_DELAY, TimeUnit.MILLISECONDS);
+                mockRunnable, asyncTaskHelper.MMODE_TASK_DELAY,
+                asyncTaskHelper.MMODE_TASK_DELAY, TimeUnit.MILLISECONDS);
         asyncTaskHelper.scheduleAsyncTask(AppcOam.RPC.maintenance_mode, mockRunnable);
         Mockito.verify(mockTask0, times(1)).cancel(true);
         Assert.assertEquals(mockTask1, asyncTaskHelper.scheduleAsyncTask(AppcOam.RPC.maintenance_mode, mockRunnable));
@@ -93,15 +94,18 @@ public class AsyncTaskHelperTest {
     }
 
     private void runTest(AppcOam.RPC rpc) {
-        Runnable mockRunnable = mock(Runnable.class);
         ScheduledFuture<?> mockTask0 = mock(ScheduledFuture.class);
         Whitebox.setInternalState(asyncTaskHelper, "backgroundOamTask", mockTask0);
+        BaseActionRunnable mockRunnable0 = mock(BaseActionRunnable.class);
+        Whitebox.setInternalState(asyncTaskHelper, "taskRunnable", mockRunnable0);
 
         ScheduledFuture<?> mockTask2 = mock(ScheduledFuture.class);
         Mockito.doReturn(mockTask2).when(mockScheduler).scheduleWithFixedDelay(
-                mockRunnable, asyncTaskHelper.COMMON_INITIAL_DELAY, asyncTaskHelper.COMMON_INTERVAL, TimeUnit.MILLISECONDS);
+                mockRunnable, asyncTaskHelper.COMMON_INITIAL_DELAY,
+                asyncTaskHelper.COMMON_INTERVAL, TimeUnit.MILLISECONDS);
         asyncTaskHelper.scheduleAsyncTask(rpc, mockRunnable);
         Mockito.verify(mockTask0, times(1)).cancel(true);
+        Mockito.verify(mockRunnable0, times(1)).abortRunnable(rpc);
         Assert.assertEquals(mockTask2, asyncTaskHelper.scheduleAsyncTask(rpc, mockRunnable));
         Assert.assertEquals("Should set backgroundOamTask", mockTask2, asyncTaskHelper.getCurrentAsyncTask());
     }
@@ -120,7 +124,8 @@ public class AsyncTaskHelperTest {
         Future<?> mockTask2 = mock(Future.class);
         asyncTaskHelper.cancelAsyncTask(mockTask2);
         Mockito.verify(mockTask2, times(1)).cancel(false);
-        Assert.assertEquals("Should not reset backgroundOamTask", mockTask, asyncTaskHelper.getCurrentAsyncTask());
+        Assert.assertEquals("Should not reset backgroundOamTask",
+                mockTask, asyncTaskHelper.getCurrentAsyncTask());
     }
 
 }