Improve fall out case handling

Update Openstack adapter to handle fall out cases
with greater stability

Issue-ID: SO-2376
Signed-off-by: Benjamin, Max (mb388a) <mb388a@att.com>
Change-Id: I20116cbd3b63bb29c9bf4ed99e92bd5f2b691da5

7 files changed, 170 insertions, 234 deletions

diff --git a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java
index b5a97f7559..75a8ca0593 100644
--- a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java
+++ b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoHeatUtils.java
@@ -89,6 +89,10 @@ import com.woorea.openstack.heat.model.Stacks;
 @Component
 public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
 
+    private static final String CREATE_FAILED = "CREATE_FAILED";
+
+    private static final String DELETE_FAILED = "DELETE_FAILED";
+
     private static final String CREATE_COMPLETE = "CREATE_COMPLETE";
 
     private static final String DELETE_COMPLETE = "DELETE_COMPLETE";
@@ -133,42 +137,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
 
     private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
 
-    /**
-     * keep this old method signature here to maintain backwards compatibility. keep others as well. this method does
-     * not include environment, files, or heatFiles
-     */
-    public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
-            String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion, int timeoutMinutes)
-            throws MsoException {
-        // Just call the new method with the environment & files variable set to null
-        return this.createStack(cloudSiteId, cloudOwner, tenantId, stackName, null, heatTemplate, stackInputs,
-                pollForCompletion, timeoutMinutes, null, null, null, true);
-    }
-
-    // This method has environment, but not files or heatFiles
-    public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
-            String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion, int timeoutMinutes,
-            String environment) throws MsoException {
-        // Just call the new method with the files/heatFiles variables set to null
-        return this.createStack(cloudSiteId, cloudOwner, tenantId, stackName, null, heatTemplate, stackInputs,
-                pollForCompletion, timeoutMinutes, environment, null, null, true);
-    }
-
-    // This method has environment and files, but not heatFiles.
-    public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
-            String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion, int timeoutMinutes,
-            String environment, Map<String, Object> files) throws MsoException {
-        return this.createStack(cloudSiteId, cloudOwner, tenantId, stackName, null, heatTemplate, stackInputs,
-                pollForCompletion, timeoutMinutes, environment, files, null, true);
-    }
 
-    // This method has environment, files, heatfiles
-    public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
-            String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion, int timeoutMinutes,
-            String environment, Map<String, Object> files, Map<String, Object> heatFiles) throws MsoException {
-        return this.createStack(cloudSiteId, cloudOwner, tenantId, stackName, null, heatTemplate, stackInputs,
-                pollForCompletion, timeoutMinutes, environment, files, heatFiles, true);
-    }
 
     /**
      * Create a new Stack in the specified cloud location and tenant. The Heat template and parameter map are passed in
@@ -211,20 +180,50 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
     @SuppressWarnings("unchecked")
     public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
             VduModelInfo vduModel, String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion,
-            int timeoutMinutes, String environment, Map<String, Object> files, Map<String, Object> heatFiles,
-            boolean backout) throws MsoException {
-
+            int timeoutMinutes, String environment, Map<String, Object> nestedTemplates, Map<String, Object> heatFiles,
+            boolean backout, boolean failIfExists) throws MsoException {
         stripMultiCloudInputs(stackInputs);
-        CreateStackParam createStack =
-                createStackParam(stackName, heatTemplate, stackInputs, timeoutMinutes, environment, files, heatFiles);
-        Stack currentStack = createStack(createStack, cloudSiteId, tenantId);
-        currentStack.setStackName(stackName);
-        if (pollForCompletion) {
-            currentStack =
+        CreateStackParam createStack = createStackParam(stackName, heatTemplate, stackInputs, timeoutMinutes,
+                environment, nestedTemplates, heatFiles);
+        Stack currentStack = queryHeatStack(stackName, cloudSiteId, tenantId);
+        if (currentStack != null) {
+            logger.debug("Existing Stack found with Status: {} ", currentStack.getStackStatus());
+            if (CREATE_COMPLETE.equals(currentStack.getStackStatus())) {
+                new StackInfoMapper(currentStack).map();
+            } else if (CREATE_IN_PROGRESS.equals(currentStack.getStackStatus())) {
+                currentStack = processCreateStack(cloudSiteId, tenantId, timeoutMinutes, backout, currentStack,
+                        createStack, true);
+            } else if (CREATE_FAILED.equals(currentStack.getStackStatus())
+                    || DELETE_FAILED.equals(currentStack.getStackStatus())) {
+                try {
                     processCreateStack(cloudSiteId, tenantId, timeoutMinutes, backout, currentStack, createStack, true);
+                } catch (MsoException e) {
+                    if (e instanceof StackCreationException) {
+                        logger.warn("Error during Stack will attempt to recreate stack");
+                        currentStack = createStack(createStack, cloudSiteId, tenantId);
+                        currentStack.setStackName(stackName);
+                        if (pollForCompletion) {
+                            currentStack = processCreateStack(cloudSiteId, tenantId, timeoutMinutes, backout,
+                                    currentStack, createStack, true);
+                        } else {
+                            currentStack = queryHeatStack(currentStack.getStackName() + "/" + currentStack.getId(),
+                                    cloudSiteId, tenantId);
+                        }
+                    } else {
+                        throw e;
+                    }
+                }
+            }
         } else {
-            currentStack =
-                    queryHeatStack(currentStack.getStackName() + "/" + currentStack.getId(), cloudSiteId, tenantId);
+            currentStack = createStack(createStack, cloudSiteId, tenantId);
+            currentStack.setStackName(stackName);
+            if (pollForCompletion) {
+                currentStack = processCreateStack(cloudSiteId, tenantId, timeoutMinutes, backout, currentStack,
+                        createStack, true);
+            } else {
+                currentStack =
+                        queryHeatStack(currentStack.getStackName() + "/" + currentStack.getId(), cloudSiteId, tenantId);
+            }
         }
         return new StackInfoMapper(currentStack).map();
     }
@@ -270,7 +269,8 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
             Stack heatStack, CreateStackParam stackCreate, boolean keyPairCleanUp) throws MsoException {
         Stack latestStack = null;
         try {
-            latestStack = pollStackForStatus(timeoutMinutes, heatStack, CREATE_IN_PROGRESS, cloudSiteId, tenantId);
+            latestStack =
+                    pollStackForStatus(timeoutMinutes, heatStack, CREATE_IN_PROGRESS, cloudSiteId, tenantId, false);
         } catch (MsoException me) {
             logger.error("Exception in Create Stack", me);
         }
@@ -280,47 +280,54 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
 
     protected Stack postProcessStackCreate(Stack stack, boolean backout, int timeoutMinutes, boolean cleanUpKeyPair,
             String cloudSiteId, String tenantId, CreateStackParam stackCreate) throws MsoException {
+        boolean stackCreationFailed = false;
+        boolean stackRollbackFailed = false;
         if (stack == null) {
             throw new StackCreationException("Unknown Error in Stack Creation");
-        }
-
-        logger.info("Performing post processing backout: {} cleanUpKeyPair: {}, stack {}", backout, cleanUpKeyPair,
-                stack);
-        if (!CREATE_COMPLETE.equals(stack.getStackStatus())) {
-            if (cleanUpKeyPair && !Strings.isNullOrEmpty(stack.getStackStatusReason())
-                    && isKeyPairFailure(stack.getStackStatusReason())) {
-                return handleKeyPairConflict(cloudSiteId, tenantId, stackCreate, timeoutMinutes, backout, stack);
-            }
-            if (!backout) {
-                logger.info("Status is not CREATE_COMPLETE, stack deletion suppressed");
-                throw new StackCreationException("Stack rollback suppressed, stack not deleted");
-            } else {
-                logger.info("Status is not CREATE_COMPLETE, stack deletion will be executed");
-                String errorMessage = "Stack Creation Failed Openstack Status: " + stack.getStackStatus()
-                        + " Status Reason: " + stack.getStackStatusReason();
-                try {
-                    Stack deletedStack = handleUnknownCreateStackFailure(stack, timeoutMinutes, cloudSiteId, tenantId);
-                    errorMessage = errorMessage + " , Rollback of Stack Creation completed with status: "
-                            + deletedStack.getStackStatus() + " Status Reason: " + deletedStack.getStackStatusReason();
-                } catch (StackRollbackException se) {
-                    logger.error("Sync Error Deleting Stack during rollback process", se);
-                    errorMessage = errorMessage + se.getMessage();
-                } catch (MsoException e) {
-                    logger.error("Sync Error Deleting Stack during rollback", e);
-
-                    errorMessage =
-                            errorMessage + " , Rollback of Stack Creation failed with sync error: " + e.getMessage();
+        } else {
+            logger.info("Performing post processing backout: {} cleanUpKeyPair: {}, stack {}", backout, cleanUpKeyPair,
+                    stack);
+            if (!CREATE_COMPLETE.equals(stack.getStackStatus())) {
+                if (cleanUpKeyPair && !Strings.isNullOrEmpty(stack.getStackStatusReason())
+                        && isKeyPairFailure(stack.getStackStatusReason())) {
+                    return handleKeyPairConflict(cloudSiteId, tenantId, stackCreate, timeoutMinutes, backout, stack);
+                }
+                if (!backout) {
+                    logger.info("Status is not CREATE_COMPLETE, stack deletion suppressed");
+                    stackCreationFailed = true;
+                    throw new StackCreationException("Stack rollback suppressed, stack not deleted",
+                            stackCreationFailed, stackRollbackFailed);
+                } else {
+                    logger.info("Status is not CREATE_COMPLETE, stack deletion will be executed");
+                    stackCreationFailed = true;
+                    String errorMessage = "Stack Creation Failed Openstack Status: " + stack.getStackStatus()
+                            + " Status Reason: " + stack.getStackStatusReason();
+                    try {
+                        Stack deletedStack =
+                                handleUnknownCreateStackFailure(stack, timeoutMinutes, cloudSiteId, tenantId);
+                        errorMessage = errorMessage + " , Rollback of Stack Creation completed with status: "
+                                + deletedStack.getStackStatus() + " Status Reason: "
+                                + deletedStack.getStackStatusReason();
+                    } catch (MsoException e) {
+                        stackRollbackFailed = true;
+                        logger.error("Sync Error Deleting Stack during rollback", e);
+                        if (e instanceof StackRollbackException) {
+                            errorMessage = errorMessage + e.getMessage();
+                        } else {
+                            errorMessage = errorMessage + " , Rollback of Stack Creation failed with sync error: "
+                                    + e.getMessage();
+                        }
+                    }
+                    throw new StackCreationException(errorMessage, stackCreationFailed, stackRollbackFailed);
                 }
-                throw new StackCreationException(errorMessage);
+            } else {
+                return stack;
             }
-        } else {
-            return stack;
         }
-
     }
 
     protected Stack pollStackForStatus(int timeoutMinutes, Stack stack, String stackStatus, String cloudSiteId,
-            String tenantId) throws MsoException {
+            String tenantId, boolean notFoundIsSuccess) throws MsoException {
         int pollingFrequency =
                 Integer.parseInt(this.environment.getProperty(createPollIntervalProp, CREATE_POLL_INTERVAL_DEFAULT));
         int pollingMultiplier =
@@ -329,9 +336,10 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
         Heat heatClient = getHeatClient(cloudSiteId, tenantId);
         while (true) {
             Stack latestStack = queryHeatStack(heatClient, stack.getStackName() + "/" + stack.getId());
-            if (latestStack != null) {
+            if (latestStack == null && notFoundIsSuccess) {
+                return null;
+            } else if (latestStack != null) {
                 statusHandler.updateStackStatus(latestStack);
-                logger.debug("Polling: {} ({})", latestStack.getStackStatus(), latestStack.getStackName());
                 if (stackStatus.equals(latestStack.getStackStatus())) {
                     if (numberOfPollingAttempts <= 0) {
                         logger.error("Polling of stack timed out with Status: {}", latestStack.getStackStatus());
@@ -342,9 +350,6 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
                 } else {
                     return latestStack;
                 }
-            } else {
-                logger.error("latestStack is null");
-                return null;
             }
         }
     }
@@ -381,17 +386,26 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
     protected Stack handleUnknownCreateStackFailure(Stack stack, int timeoutMinutes, String cloudSiteId,
             String tenantId) throws MsoException {
         if (stack != null && !Strings.isNullOrEmpty(stack.getStackName()) && !Strings.isNullOrEmpty(stack.getId())) {
-            OpenStackRequest<Void> request = getHeatClient(cloudSiteId, tenantId).getStacks()
-                    .deleteByName(stack.getStackName() + "/" + stack.getId());
-            executeAndRecordOpenstackRequest(request);
-            Stack currentStack = pollStackForStatus(timeoutMinutes, stack, DELETE_IN_PROGRESS, cloudSiteId, tenantId);
-            postProcessStackDelete(currentStack);
-            return currentStack;
+            return deleteStack(stack, timeoutMinutes, cloudSiteId, tenantId);
         } else {
             throw new StackCreationException("Cannot Find Stack Name or Id");
         }
     }
 
+    private Stack deleteStack(Stack stack, int timeoutMinutes, String cloudSiteId, String tenantId)
+            throws MsoException {
+        OpenStackRequest<Void> request = getHeatClient(cloudSiteId, tenantId).getStacks()
+                .deleteByName(stack.getStackName() + "/" + stack.getId());
+        executeAndRecordOpenstackRequest(request);
+        Stack currentStack = pollStackForStatus(timeoutMinutes, stack, DELETE_IN_PROGRESS, cloudSiteId, tenantId, true);
+        if (currentStack == null) {
+            return currentStack;
+        } else {
+            postProcessStackDelete(currentStack);
+        }
+        return currentStack;
+    }
+
     protected void postProcessStackDelete(Stack stack) throws MsoException {
         logger.info("Performing post processing on delete stack {}", stack);
         if (stack != null && !Strings.isNullOrEmpty(stack.getStackStatus())) {
@@ -453,7 +467,6 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
         // Query the Stack.
         // An MsoException will propagate transparently to the caller.
         Stack heatStack = queryHeatStack(heatClient, stackName);
-
         if (heatStack == null) {
             // Stack does not exist. Return a StackInfo with status NOTFOUND
             return new StackInfo(stackName, HeatStatus.NOTFOUND);
@@ -484,114 +497,16 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
      * @throws MsoCloudSiteNotFound
      */
     public StackInfo deleteStack(String tenantId, String cloudOwner, String cloudSiteId, String stackName,
-            boolean pollForCompletion) throws MsoException {
-        Heat heatClient = null;
-        try {
-            heatClient = getHeatClient(cloudSiteId, tenantId);
-        } catch (MsoTenantNotFound e) {
-            logger.debug("Tenant with id " + tenantId + "not found.", e);
-            return new StackInfo(stackName, HeatStatus.NOTFOUND);
-        } catch (MsoException me) {
-            logger.error("{} {} Openstack Exception on Token request: ", MessageEnum.RA_CONNECTION_EXCEPTION,
-                    ErrorCode.AvailabilityError.getValue(), me);
-            me.addContext(DELETE_STACK);
-            throw me;
-        }
-
-        // OK if stack not found, perform a query first
-        Stack heatStack = queryHeatStack(heatClient, stackName);
-        if (heatStack == null || DELETE_COMPLETE.equals(heatStack.getStackStatus())) {
-            // Not found. Return a StackInfo with status NOTFOUND
-            return new StackInfo(stackName, HeatStatus.NOTFOUND);
-        }
-
-        // Use canonical name "<stack name>/<stack-id>" to delete.
-        // Otherwise, deletion by name returns a 302 redirect.
-        // NOTE: This is specific to the v1 Orchestration API.
-        String canonicalName = heatStack.getStackName() + "/" + heatStack.getId();
-
-        try {
-            OpenStackRequest<Void> request = null;
-            if (null != heatClient) {
-                request = heatClient.getStacks().deleteByName(canonicalName);
-            } else {
-                logger.debug("Heat Client is NULL");
-            }
-            executeAndRecordOpenstackRequest(request);
-        } catch (OpenStackResponseException e) {
-            if (e.getStatus() == 404) {
-                // Not found. We are OK with this. Return a StackInfo with status NOTFOUND
-                return new StackInfo(stackName, HeatStatus.NOTFOUND);
-            } else {
-                // Convert the OpenStackResponseException to an MsoOpenstackException
-                throw heatExceptionToMsoException(e, DELETE_STACK);
-            }
-        } catch (OpenStackConnectException e) {
-            // Error connecting to Openstack instance. Convert to an MsoException
-            throw heatExceptionToMsoException(e, DELETE_STACK);
-        } catch (RuntimeException e) {
-            // Catch-all
-            throw runtimeExceptionToMsoException(e, DELETE_STACK);
-        }
-
-        // Requery the stack for current status.
-        // It will probably still exist with "DELETE_IN_PROGRESS" status.
-        heatStack = queryHeatStack(heatClient, canonicalName);
-        statusHandler.updateStackStatus(heatStack);
-        if (pollForCompletion) {
-            int pollInterval = Integer
-                    .parseInt(this.environment.getProperty(deletePollIntervalProp, "" + DELETE_POLL_INTERVAL_DEFAULT));
-            int pollTimeout = Integer
-                    .parseInt(this.environment.getProperty(deletePollTimeoutProp, "" + DELETE_POLL_INTERVAL_DEFAULT));
-            statusHandler.updateStackStatus(heatStack);
-            // When querying by canonical name, Openstack returns DELETE_COMPLETE status
-            // instead of "404" (which would result from query by stack name).
-            while (heatStack != null && !DELETE_COMPLETE.equals(heatStack.getStackStatus())) {
-                logger.debug("Stack status: {}", heatStack.getStackStatus());
-
-                if ("DELETE_FAILED".equals(heatStack.getStackStatus())) {
-                    // Throw a 'special case' of MsoOpenstackException to report the Heat status
-                    String error = "Stack delete error (" + heatStack.getStackStatus() + "): "
-                            + heatStack.getStackStatusReason();
-                    MsoOpenstackException me = new MsoOpenstackException(0, "", error);
-                    me.addContext(DELETE_STACK);
-
-                    // Alarm this condition, stack deletion failed
-
-
-                    throw me;
-                }
-
-                if (pollTimeout <= 0) {
-                    logger.error("{} Cloud site: {} Tenant: {} Stack: {} Stack status: {} {} Delete Stack Timeout",
-                            MessageEnum.RA_DELETE_STACK_TIMEOUT, cloudSiteId, tenantId, stackName,
-                            heatStack.getStackStatus(), ErrorCode.AvailabilityError.getValue());
-
-                    // Throw a 'special case' of MsoOpenstackException to report the Heat status
-                    MsoOpenstackException me = new MsoOpenstackException(0, "", "Stack Deletion Timeout");
-                    me.addContext(DELETE_STACK);
-
-                    // Alarm this condition, stack deletion failed
-
-
-                    throw me;
-                }
-
-                sleep(pollInterval * 1000L);
-
-                pollTimeout -= pollInterval;
-                logger.debug("pollTimeout remaining: {}", pollTimeout);
-
-                heatStack = queryHeatStack(heatClient, canonicalName);
-            }
-
-            // The stack is gone when this point is reached
+            boolean pollForCompletion, int timeoutMinutes) throws MsoException {
+        Stack currentStack = queryHeatStack(stackName, cloudSiteId, tenantId);
+        if (currentStack == null || DELETE_COMPLETE.equals(currentStack.getStackStatus())) {
             return new StackInfo(stackName, HeatStatus.NOTFOUND);
+        } else {
+            currentStack = deleteStack(currentStack, timeoutMinutes, cloudSiteId, tenantId);
+            StackInfo stackInfo = new StackInfoMapper(currentStack).map();
+            stackInfo.setName(stackName);
+            return stackInfo;
         }
-        // Return the current status (if not polling, the delete may still be in progress)
-        StackInfo stackInfo = new StackInfoMapper(heatStack).map();
-        stackInfo.setName(stackName);
-        return stackInfo;
     }
 
     /**
@@ -734,7 +649,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
             if (!inputs.containsKey(key)) {
                 Object obj = otherStackOutputs.get(key);
                 if (obj instanceof String) {
-                    inputs.put(key, (String) otherStackOutputs.get(key));
+                    inputs.put(key, otherStackOutputs.get(key));
                 } else if (obj instanceof JsonNode) {
                     // This is a bit of mess - but I think it's the least impacting
                     // let's convert it BACK to a string - then it will get converted back later
@@ -1115,11 +1030,11 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
         }
 
         try {
-            StackInfo stackInfo =
-                    createStack(cloudSiteId, cloudOwner, tenantId, instanceName, vduModel, heatTemplate, inputs, true, // poll
-                                                                                                                       // for
-                                                                                                                       // completion
-                            vduModel.getTimeoutMinutes(), heatEnvironment, nestedTemplates, files, rollbackOnFailure);
+            StackInfo stackInfo = createStack(cloudSiteId, cloudOwner, tenantId, instanceName, vduModel, heatTemplate,
+                    inputs, true, // poll
+                                  // for
+                                  // completion
+                    vduModel.getTimeoutMinutes(), heatEnvironment, nestedTemplates, files, rollbackOnFailure, false);
 
             // Populate a vduInstance from the StackInfo
             return stackInfoToVduInstance(stackInfo);
@@ -1160,7 +1075,7 @@ public class MsoHeatUtils extends MsoCommonUtils implements VduPlugin {
 
         try {
             // Delete the Heat stack
-            StackInfo stackInfo = deleteStack(tenantId, cloudOwner, cloudSiteId, instanceId, true);
+            StackInfo stackInfo = deleteStack(tenantId, cloudOwner, cloudSiteId, instanceId, true, timeoutMinutes);
 
             // Populate a VduInstance based on the deleted Cloudify Deployment object
             VduInstance vduInstance = stackInfoToVduInstance(stackInfo);
diff --git a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoMulticloudUtils.java b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoMulticloudUtils.java
index 1701cda98e..3eb3fe188d 100644
--- a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoMulticloudUtils.java
+++ b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/MsoMulticloudUtils.java
@@ -135,7 +135,7 @@ public class MsoMulticloudUtils extends MsoHeatUtils implements VduPlugin {
     public StackInfo createStack(String cloudSiteId, String cloudOwner, String tenantId, String stackName,
             VduModelInfo vduModel, String heatTemplate, Map<String, ?> stackInputs, boolean pollForCompletion,
             int timeoutMinutes, String environment, Map<String, Object> files, Map<String, Object> heatFiles,
-            boolean backout) throws MsoException {
+            boolean backout, boolean failIfExists) throws MsoException {
 
         logger.trace("Started MsoMulticloudUtils.createStack");
 
@@ -881,11 +881,11 @@ public class MsoMulticloudUtils extends MsoHeatUtils implements VduPlugin {
         }
 
         try {
-            StackInfo stackInfo =
-                    createStack(cloudSiteId, cloudOwner, tenantId, instanceName, vduModel, heatTemplate, inputs, true, // poll
-                                                                                                                       // for
-                                                                                                                       // completion
-                            vduModel.getTimeoutMinutes(), heatEnvironment, nestedTemplates, files, rollbackOnFailure);
+            StackInfo stackInfo = createStack(cloudSiteId, cloudOwner, tenantId, instanceName, vduModel, heatTemplate,
+                    inputs, true, // poll
+                                  // for
+                                  // completion
+                    vduModel.getTimeoutMinutes(), heatEnvironment, nestedTemplates, files, rollbackOnFailure, false);
             // Populate a vduInstance from the StackInfo
             return stackInfoToVduInstance(stackInfo);
         } catch (Exception e) {
@@ -957,6 +957,7 @@ public class MsoMulticloudUtils extends MsoHeatUtils implements VduPlugin {
     /*
      * Convert the local DeploymentInfo object (Cloudify-specific) to a generic VduInstance object
      */
+    @Override
     protected VduInstance stackInfoToVduInstance(StackInfo stackInfo) {
         VduInstance vduInstance = new VduInstance();
 
diff --git a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/StackCreationException.java b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/StackCreationException.java
index e3936e31f7..ab9c8a6dee 100644
--- a/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/StackCreationException.java
+++ b/adapters/mso-adapter-utils/src/main/java/org/onap/so/openstack/utils/StackCreationException.java
@@ -24,8 +24,22 @@ import org.onap.so.openstack.exceptions.MsoException;
 
 public class StackCreationException extends MsoException {
 
+    /**
+     * 
+     */
+    private static final long serialVersionUID = 6221508301589330950L;
+
+    boolean stackCreationFailed = false;
+    boolean stackRollbackFailed = false;
+
     public StackCreationException(String error) {
         super(error);
     }
 
+    public StackCreationException(String error, boolean stackCreationFailed, boolean stackRollbackFailed) {
+        super(error);
+        this.stackRollbackFailed = stackRollbackFailed;
+        this.stackCreationFailed = stackCreationFailed;
+    }
+
 }
diff --git a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsITTest.java b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsITTest.java
index 7e783aa7a4..86b1413291 100644
--- a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsITTest.java
+++ b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsITTest.java
@@ -133,7 +133,7 @@ public class MsoHeatUtilsITTest extends BaseTest {
     @Test
     public void deleteVduTest() throws Exception {
         VduInstance expected = new VduInstance();
-        expected.setVduInstanceId("instanceId");
+        expected.setVduInstanceId("name/stackId");
         expected.setVduInstanceName("instanceId");
         VduStatus status = new VduStatus();
         status.setState(VduStateType.DELETED);
@@ -209,8 +209,8 @@ public class MsoHeatUtilsITTest extends BaseTest {
         StubOpenStack.mockOpenStackResponseAccess(wireMockServer, wireMockPort);
         StubOpenStack.mockOpenStackPostStack_200(wireMockServer, "OpenstackResponse_Stack_Created.json");
         StubOpenStack.mockOpenStackGet(wireMockServer, "TEST-stack/stackId");
-        StackInfo stackInfo = heatUtils.createStack(cloudSite.getId(), "CloudOwner", "tenantId", "TEST-stack",
-                "TEST-heat", new HashMap<>(), false, 1, "TEST-env", new HashMap<>(), new HashMap<>());
+        StackInfo stackInfo = heatUtils.createStack(cloudSite.getId(), "CloudOwner", "tenantId", "TEST-stack", null,
+                "TEST-heat", new HashMap<>(), false, 1, "TEST-env", new HashMap<>(), new HashMap<>(), false, false);
         assertNotNull(stackInfo);
     }
 
diff --git a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsTest.java b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsTest.java
index e68364eab8..92cf96ed16 100644
--- a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsTest.java
+++ b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoHeatUtilsTest.java
@@ -124,7 +124,7 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         latestStack.setStackStatusReason("Stack Finished");
         doReturn(latestStack).when(heatUtils).queryHeatStack(isA(Heat.class), eq("stackName/id"));
         doReturn(heatClient).when(heatUtils).getHeatClient(cloudSiteId, tenantId);
-        Stack actual = heatUtils.pollStackForStatus(1, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+        Stack actual = heatUtils.pollStackForStatus(1, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId, false);
         Mockito.verify(stackStatusHandler, times(1)).updateStackStatus(latestStack);
         Mockito.verify(heatUtils, times(1)).queryHeatStack(isA(Heat.class), eq("stackName/id"));
         assertEquals(true, actual != null);
@@ -141,7 +141,7 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         doNothing().when(stackStatusHandler).updateStackStatus(stack);
         doReturn(stack).when(heatUtils).queryHeatStack(isA(Heat.class), eq("stackName/id"));
         doReturn(heatClient).when(heatUtils).getHeatClient(cloudSiteId, tenantId);
-        Stack actual = heatUtils.pollStackForStatus(1, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+        Stack actual = heatUtils.pollStackForStatus(1, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId, false);
         Mockito.verify(stackStatusHandler, times(1)).updateStackStatus(stack);
         Mockito.verify(heatUtils, times(1)).queryHeatStack(isA(Heat.class), eq("stackName/id"));
         assertEquals(true, actual != null);
@@ -259,11 +259,12 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         doReturn(stackResource).when(heatClient).getStacks();
         doReturn(mockDeleteStack).when(stackResource).deleteByName("stackName/id");
         doReturn(deletedStack).when(heatUtils).pollStackForStatus(120, stack, "DELETE_IN_PROGRESS", cloudSiteId,
-                tenantId);
+                tenantId, true);
 
         heatUtils.handleUnknownCreateStackFailure(stack, 120, cloudSiteId, tenantId);
         Mockito.verify(heatUtils, times(1)).executeAndRecordOpenstackRequest(mockDeleteStack);
-        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "DELETE_IN_PROGRESS", cloudSiteId, tenantId);
+        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "DELETE_IN_PROGRESS", cloudSiteId, tenantId,
+                true);
         Mockito.verify(heatUtils, times(1)).postProcessStackDelete(deletedStack);
     }
 
@@ -357,12 +358,13 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         createStackParam.setStackName("stackName");
 
         doReturn(createdStack).when(heatUtils).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId,
-                tenantId);
+                tenantId, false);
         doReturn(createdStack).when(heatUtils).postProcessStackCreate(createdStack, true, 120, true, cloudSiteId,
                 tenantId, createStackParam);
 
         heatUtils.processCreateStack(cloudSiteId, tenantId, 120, true, stack, createStackParam, true);
-        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId,
+                false);
         Mockito.verify(heatUtils, times(1)).postProcessStackCreate(createdStack, true, 120, true, cloudSiteId, tenantId,
                 createStackParam);
     }
@@ -387,11 +389,12 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         createStackParam.setStackName("stackName");
 
         doThrow(new StackCreationException("Error")).when(heatUtils).pollStackForStatus(120, stack,
-                "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+                "CREATE_IN_PROGRESS", cloudSiteId, tenantId, false);
         exceptionRule.expect(MsoException.class);
         exceptionRule.expectMessage("Error");
         heatUtils.processCreateStack(cloudSiteId, tenantId, 120, true, stack, createStackParam, true);
-        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId,
+                false);
         Mockito.verify(heatUtils, times(1)).postProcessStackCreate(stack, true, 120, true, cloudSiteId, tenantId,
                 createStackParam);
     }
@@ -527,12 +530,13 @@ public class MsoHeatUtilsTest extends MsoHeatUtils {
         createStackParam.setStackName("stackName");
 
         doThrow(new StackCreationException("Error")).when(heatUtils).pollStackForStatus(120, stack,
-                "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+                "CREATE_IN_PROGRESS", cloudSiteId, tenantId, false);
 
         exceptionRule.expect(MsoException.class);
         exceptionRule.expectMessage("Error");
         heatUtils.processCreateStack(cloudSiteId, tenantId, 120, false, stack, createStackParam, true);
-        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId);
+        Mockito.verify(heatUtils, times(1)).pollStackForStatus(120, stack, "CREATE_IN_PROGRESS", cloudSiteId, tenantId,
+                false);
         Mockito.verify(heatUtils, times(0)).handleUnknownCreateStackFailure(stack, 120, cloudSiteId, tenantId);
     }
 
diff --git a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoMulticloudUtilsTest.java b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoMulticloudUtilsTest.java
index 995db522e9..44a9bc3e10 100644
--- a/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoMulticloudUtilsTest.java
+++ b/adapters/mso-adapter-utils/src/test/java/org/onap/so/openstack/utils/MsoMulticloudUtilsTest.java
@@ -24,8 +24,8 @@ import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
 import static com.github.tomakehurst.wiremock.client.WireMock.delete;
 import static com.github.tomakehurst.wiremock.client.WireMock.get;
 import static com.github.tomakehurst.wiremock.client.WireMock.post;
-import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -95,9 +95,9 @@ public class MsoMulticloudUtilsTest extends BaseTest {
         wireMockServer.stubFor(get(urlEqualTo(MULTICLOUD_GET_PATH)).inScenario("CREATE").whenScenarioStateIs("UPDATING")
                 .willReturn(aResponse().withHeader("Content-Type", "application/json")
                         .withBodyFile("MulticloudGetUpdateResponse.json").withStatus(HttpStatus.SC_OK)));
-        StackInfo result =
-                multicloudUtils.createStack("MTN14", "CloudOwner", "TEST-tenant", "TEST-stack", new VduModelInfo(),
-                        "TEST-heat", new HashMap<>(), true, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false);
+        StackInfo result = multicloudUtils.createStack("MTN14", "CloudOwner", "TEST-tenant", "TEST-stack",
+                new VduModelInfo(), "TEST-heat", new HashMap<>(), true, 200, "TEST-env", new HashMap<>(),
+                new HashMap<>(), false, false);
         wireMockServer.resetScenarios();
         assertNotNull(result);
         assertEquals("TEST-stack", result.getName());
@@ -155,7 +155,8 @@ public class MsoMulticloudUtilsTest extends BaseTest {
             cloudSite.setIdentityService(new CloudIdentity());
             when(cloudConfigMock.getCloudSite("MTN13")).thenReturn(Optional.of(cloudSite));
             multicloudUtilsMock.createStack("MNT14", "CloudOwner", "TEST-tenant", "TEST-stack", new VduModelInfo(),
-                    "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false);
+                    "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false,
+                    false);
         } catch (MsoException e) {
             assertEquals("0 : Multicloud client could not be initialized", e.toString());
             return;
@@ -169,7 +170,8 @@ public class MsoMulticloudUtilsTest extends BaseTest {
             wireMockServer.stubFor(post(urlPathEqualTo(MULTICLOUD_CREATE_PATH)).willReturn(
                     aResponse().withHeader("Content-Type", "application/json").withStatus(HttpStatus.SC_BAD_REQUEST)));
             multicloudUtils.createStack("MTN14", "CloudOwner", "TEST-tenant", "TEST-stack", new VduModelInfo(),
-                    "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false);
+                    "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false,
+                    false);
         } catch (MsoException e) {
             assertEquals("0 : Bad Request", e.toString());
             return;
@@ -181,9 +183,9 @@ public class MsoMulticloudUtilsTest extends BaseTest {
     public void createStackEmptyResponseEntity() throws MsoException {
         wireMockServer.stubFor(post(urlPathEqualTo(MULTICLOUD_CREATE_PATH)).willReturn(
                 aResponse().withHeader("Content-Type", "application/json").withStatus(HttpStatus.SC_CREATED)));
-        StackInfo result =
-                multicloudUtils.createStack("MTN14", "CloudOwner", "TEST-tenant", "TEST-stack", new VduModelInfo(),
-                        "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(), new HashMap<>(), false);
+        StackInfo result = multicloudUtils.createStack("MTN14", "CloudOwner", "TEST-tenant", "TEST-stack",
+                new VduModelInfo(), "TEST-heat", new HashMap<>(), false, 200, "TEST-env", new HashMap<>(),
+                new HashMap<>(), false, false);
         assertNotNull(result);
         assertEquals("TEST-stack/", result.getName());
     }
diff --git a/adapters/mso-adapter-utils/src/test/resources/__files/OpenstackResponse_Stack_DeleteComplete.json b/adapters/mso-adapter-utils/src/test/resources/__files/OpenstackResponse_Stack_DeleteComplete.json
index a26a551f7d..ea05b64e08 100644
--- a/adapters/mso-adapter-utils/src/test/resources/__files/OpenstackResponse_Stack_DeleteComplete.json
+++ b/adapters/mso-adapter-utils/src/test/resources/__files/OpenstackResponse_Stack_DeleteComplete.json
@@ -3,7 +3,7 @@
 		"description": null,
 		"links": null,
 		"stack_status_reason": null,
-		"stack_name": null,
+		"stack_name": "name",
 		"updated_time": null,
 		"creation_time": null,
 		"stack_status": "DELETE_COMPLETE",