Improve fall out case handling

Update Openstack adapter to handle fall out cases
with greater stability

Issue-ID: SO-2376
Signed-off-by: Benjamin, Max (mb388a) <mb388a@att.com>
Change-Id: I20116cbd3b63bb29c9bf4ed99e92bd5f2b691da5

6 files changed, 65 insertions, 267 deletions

diff --git a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/network/MsoNetworkAdapterImpl.java b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/network/MsoNetworkAdapterImpl.java
index 013c7f8a84..52b97da61a 100644
--- a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/network/MsoNetworkAdapterImpl.java
+++ b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/network/MsoNetworkAdapterImpl.java
@@ -24,8 +24,6 @@
 
 package org.onap.so.adapters.network;
 
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -33,7 +31,6 @@ import java.util.Map;
 import java.util.Optional;
 import javax.jws.WebService;
 import javax.xml.ws.Holder;
-import org.onap.so.logger.LoggingAnchor;
 import org.onap.so.adapters.network.beans.ContrailPolicyRef;
 import org.onap.so.adapters.network.beans.ContrailPolicyRefSeq;
 import org.onap.so.adapters.network.beans.ContrailSubnet;
@@ -51,6 +48,7 @@ import org.onap.so.db.catalog.data.repository.NetworkResourceRepository;
 import org.onap.so.db.catalog.utils.MavenLikeVersioning;
 import org.onap.so.entity.MsoRequest;
 import org.onap.so.logger.ErrorCode;
+import org.onap.so.logger.LoggingAnchor;
 import org.onap.so.logger.MessageEnum;
 import org.onap.so.openstack.beans.HeatStatus;
 import org.onap.so.openstack.beans.NetworkInfo;
@@ -74,6 +72,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 @Component
 @Transactional
@@ -427,7 +427,8 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                 if (backout == null)
                     backout = true;
                 heatStack = heat.createStack(cloudSiteId, CLOUD_OWNER, tenantId, networkName, null, template,
-                        stackParams, true, heatTemplate.getTimeoutMinutes(), null, null, null, backout.booleanValue());
+                        stackParams, true, heatTemplate.getTimeoutMinutes(), null, null, null, backout.booleanValue(),
+                        failIfExists);
             } catch (MsoException me) {
                 me.addContext(CREATE_NETWORK_CONTEXT);
                 logger.error("{} {} Exception creating network type {} in {}/{} ", MessageEnum.RA_CREATE_NETWORK_EXC,
@@ -1015,13 +1016,7 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
     @Override
     public void deleteNetwork(String cloudSiteId, String tenantId, String networkType, String modelCustomizationUuid,
             String networkId, MsoRequest msoRequest, Holder<Boolean> networkDeleted) throws NetworkException {
-
         logger.debug("*** DELETE Network adapter with Network: {} in {}/{}", networkId, cloudSiteId, tenantId);
-
-        // Will capture execution time for metrics
-        long startTime = System.currentTimeMillis();
-
-
         if (commonUtils.isNullOrEmpty(cloudSiteId) || commonUtils.isNullOrEmpty(tenantId)
                 || commonUtils.isNullOrEmpty(networkId)) {
             String error = "Missing mandatory parameter cloudSiteId, tenantId or networkId";
@@ -1043,20 +1038,15 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                 networkResource = nrc.getNetworkResource();
             }
         }
+
         String mode = "";
         if (networkResource != null) {
-            logger.debug(LOG_DEBUG_MSG, networkResource);
-
+            logger.debug(LOG_DEBUG_MSG, networkResource.toString());
             mode = networkResource.getOrchestrationMode();
         }
 
         if (NEUTRON_MODE.equals(mode)) {
-
-            // Use MsoNeutronUtils for all NEUTRON commands
-            long deleteNetworkStarttime = System.currentTimeMillis();
             try {
-                // The deleteNetwork function in MsoNeutronUtils returns success if the network
-                // was not found. So don't bother to query first.
                 boolean deleted = neutron.deleteNetwork(networkId, tenantId, cloudSiteId);
                 networkDeleted.value = deleted;
             } catch (MsoException me) {
@@ -1065,21 +1055,10 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                         ErrorCode.DataError.getValue(), networkId, cloudSiteId, tenantId, me);
                 throw new NetworkException(me);
             }
-        } else { // DEFAULT to ("HEAT".equals (mode))
-            long deleteStackStarttime = System.currentTimeMillis();
-
+        } else {
             try {
-                // The deleteStack function in MsoHeatUtils returns NOTFOUND if the stack was not found or if the stack
-                // was deleted.
-                // So query first to report back if stack WAS deleted or just NOTOFUND
-                StackInfo heatStack = null;
-                heatStack = heat.queryStack(cloudSiteId, CLOUD_OWNER, tenantId, networkId);
-                if (heatStack != null && heatStack.getStatus() != HeatStatus.NOTFOUND) {
-                    heat.deleteStack(tenantId, CLOUD_OWNER, cloudSiteId, networkId, true);
-                    networkDeleted.value = true;
-                } else {
-                    networkDeleted.value = false;
-                }
+                heat.deleteStack(tenantId, CLOUD_OWNER, cloudSiteId, networkId, true, 120);
+                networkDeleted.value = true;
             } catch (MsoException me) {
                 me.addContext("DeleteNetwork");
                 logger.error("{} {} Delete Network (heat): {} in {}/{} ", MessageEnum.RA_DELETE_NETWORK_EXC,
@@ -1087,10 +1066,6 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                 throw new NetworkException(me);
             }
         }
-
-
-        // On success, nothing is returned.
-        return;
     }
 
     /**
@@ -1103,9 +1078,6 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
      */
     @Override
     public void rollbackNetwork(NetworkRollback rollback) throws NetworkException {
-        // Will capture execution time for metrics
-        long startTime = System.currentTimeMillis();
-
         if (rollback == null) {
             logger.error("{} {} rollback is null", MessageEnum.RA_ROLLBACK_NULL, ErrorCode.DataError.getValue());
             return;
@@ -1119,8 +1091,6 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
         String modelCustomizationUuid = rollback.getModelCustomizationUuid();
 
         logger.debug("*** ROLLBACK Network {} in {}/{}", networkId, cloudSiteId, tenantId);
-
-
         // Retrieve the Network Resource definition
         NetworkResource networkResource = null;
         if (commonUtils.isNullOrEmpty(modelCustomizationUuid)) {
@@ -1138,13 +1108,8 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
         }
 
         if (rollback.getNetworkCreated()) {
-            // Rolling back a newly created network, so delete it.
             if (NEUTRON_MODE.equals(mode)) {
-                // Use MsoNeutronUtils for all NEUTRON commands
-                long deleteNetworkStarttime = System.currentTimeMillis();
                 try {
-                    // The deleteNetwork function in MsoNeutronUtils returns success if the network
-                    // was not found. So don't bother to query first.
                     neutron.deleteNetwork(networkId, tenantId, cloudSiteId);
                 } catch (MsoException me) {
                     me.addContext("RollbackNetwork");
@@ -1153,12 +1118,9 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                             cloudSiteId, tenantId, me);
                     throw new NetworkException(me);
                 }
-            } else { // DEFAULT to if ("HEAT".equals (mode))
-                long deleteStackStarttime = System.currentTimeMillis();
+            } else {
                 try {
-                    // The deleteStack function in MsoHeatUtils returns success if the stack
-                    // was not found. So don't bother to query first.
-                    heat.deleteStack(tenantId, CLOUD_OWNER, cloudSiteId, networkId, true);
+                    heat.deleteStack(tenantId, CLOUD_OWNER, cloudSiteId, networkId, true, 120);
                 } catch (MsoException me) {
                     me.addContext("RollbackNetwork");
                     logger.error("{} {} Exception - Rollback Network (heat): {} in {}/{} ",
@@ -1168,8 +1130,6 @@ public class MsoNetworkAdapterImpl implements MsoNetworkAdapter {
                 }
             }
         }
-
-        return;
     }
 
     private String validateNetworkParams(NetworkType neutronNetworkType, String networkName, String physicalNetwork,
diff --git a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/MsoVnfAdapterImpl.java b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/MsoVnfAdapterImpl.java
index af2fa24ff9..2c78e593a1 100644
--- a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/MsoVnfAdapterImpl.java
+++ b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/MsoVnfAdapterImpl.java
@@ -36,7 +36,6 @@ import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import javax.jws.WebService;
 import javax.xml.ws.Holder;
-import org.onap.so.logger.LoggingAnchor;
 import org.apache.commons.collections.CollectionUtils;
 import org.onap.so.adapters.valet.GenericValetResponse;
 import org.onap.so.adapters.valet.ValetClient;
@@ -47,7 +46,6 @@ import org.onap.so.adapters.valet.beans.ValetDeleteResponse;
 import org.onap.so.adapters.valet.beans.ValetRollbackResponse;
 import org.onap.so.adapters.valet.beans.ValetStatus;
 import org.onap.so.adapters.valet.beans.ValetUpdateResponse;
-import org.onap.so.adapters.vnf.exceptions.VnfAlreadyExists;
 import org.onap.so.adapters.vnf.exceptions.VnfException;
 import org.onap.so.adapters.vnf.exceptions.VnfNotFound;
 import org.onap.so.client.aai.AAIResourcesClient;
@@ -68,6 +66,7 @@ import org.onap.so.entity.MsoRequest;
 import org.onap.so.heatbridge.HeatBridgeApi;
 import org.onap.so.heatbridge.HeatBridgeImpl;
 import org.onap.so.logger.ErrorCode;
+import org.onap.so.logger.LoggingAnchor;
 import org.onap.so.logger.MessageEnum;
 import org.onap.so.openstack.beans.HeatStatus;
 import org.onap.so.openstack.beans.StackInfo;
@@ -80,6 +79,10 @@ import org.onap.so.openstack.exceptions.MsoHeatNotFoundException;
 import org.onap.so.openstack.utils.MsoHeatEnvironmentEntry;
 import org.onap.so.openstack.utils.MsoHeatUtils;
 import org.onap.so.openstack.utils.MsoHeatUtilsWithUpdate;
+import org.openstack4j.model.compute.Flavor;
+import org.openstack4j.model.compute.Image;
+import org.openstack4j.model.compute.Server;
+import org.openstack4j.model.heat.Resource;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -89,10 +92,6 @@ import org.springframework.transaction.annotation.Transactional;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import org.openstack4j.model.compute.Flavor;
-import org.openstack4j.model.compute.Image;
-import org.openstack4j.model.compute.Server;
-import org.openstack4j.model.heat.Resource;
 
 @WebService(serviceName = "VnfAdapter", endpointInterface = "org.onap.so.adapters.vnf.MsoVnfAdapter",
         targetNamespace = "http://org.onap.so/vnf")
@@ -133,7 +132,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
     @Autowired
     private MsoHeatUtilsWithUpdate heatU;
     @Autowired
-    private MsoHeatUtils heat;
+    private MsoHeatUtils msoHeatUtils;
     @Autowired
     private ValetClient vci;
 
@@ -264,7 +263,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
 
         StackInfo heatStack;
         try {
-            heatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, vnfName);
+            heatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, vnfName);
         } catch (MsoException me) {
             me.addContext("QueryVNF");
             // Failed to query the Stack due to an openstack exception.
@@ -317,7 +316,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         logger.debug("Deleting VNF {} in {}", vnfName, cloudSiteId + "/" + tenantId);
 
         try {
-            heat.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfName, true);
+            msoHeatUtils.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfName, true, 118);
         } catch (MsoException me) {
             me.addContext(DELETE_VNF);
             // Failed to query the Stack due to an openstack exception.
@@ -360,7 +359,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         // of NOTFOUND (on success) or FAILED (on error). Also, MsoOpenstackException
         // could be thrown.
         try {
-            heat.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfId, true);
+            msoHeatUtils.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfId, true, 118);
         } catch (MsoException me) {
             // Failed to rollback the Stack due to an openstack exception.
             // Convert to a generic VnfException
@@ -638,96 +637,13 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
             callHeatbridge(baseVfHeatStackId);
         }
 
-        StackInfo heatStack = null;
-        try {
-            if (heat != null) {
-                heatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, vfModuleName);
-            } else {
-                throw new MsoHeatNotFoundException();
-            }
-        } catch (MsoException me) {
-            String error = "Create VF Module: Query " + vfModuleName + " in " + cloudOwner + "/" + cloudSiteId + "/"
-                    + tenantId + ": " + me;
-            logger.error(LoggingAnchor.NINE, MessageEnum.RA_QUERY_VNF_ERR.toString(), vfModuleName, cloudOwner,
-                    cloudSiteId, tenantId, OPENSTACK, QUERY_STACK, ErrorCode.DataError.getValue(),
-                    "Exception - " + QUERY_STACK, me);
-            logger.debug(error);
-            // Failed to query the Stack due to an openstack exception.
-            // Convert to a generic VnfException
-            me.addContext(CREATE_VFM_MODULE);
-            throw new VnfException(me);
-        }
-        // more precise handling/messaging if the stack already exists
-        if (heatStack != null && heatStack.getStatus() != HeatStatus.NOTFOUND) {
-            // INIT, CREATED, NOTFOUND, FAILED, BUILDING, DELETING, UNKNOWN, UPDATING, UPDATED
-            HeatStatus status = heatStack.getStatus();
-            if (status == HeatStatus.INIT || status == HeatStatus.BUILDING || status == HeatStatus.DELETING
-                    || status == HeatStatus.UPDATING) {
-                // fail - it's in progress - return meaningful error
-                String error = CREATE_VF_STACK + " " + vfModuleName + " already exists and has status "
-                        + status.toString() + " in " + cloudOwner + "/" + cloudSiteId + "/" + tenantId
-                        + "; please wait for it to complete, or fix manually.";
-                logger.error(LoggingAnchor.NINE, MessageEnum.RA_VNF_ALREADY_EXIST.toString(), vfModuleName, cloudOwner,
-                        cloudSiteId, tenantId, OPENSTACK, QUERY_STACK, ErrorCode.DataError.getValue(),
-                        STACK + " " + vfModuleName + " already exists");
-                logger.debug(error);
-                throw new VnfAlreadyExists(vfModuleName, cloudOwner, cloudSiteId, tenantId,
-                        heatStack.getCanonicalName());
-            }
-            if (status == HeatStatus.FAILED) {
-                // fail - it exists and is in a FAILED state
-                String error = CREATE_VF_STACK + " " + vfModuleName + " already exists and is in FAILED state in "
-                        + cloudOwner + "/" + cloudSiteId + "/" + tenantId + "; requires manual intervention.";
-                logger.error(LoggingAnchor.NINE, MessageEnum.RA_VNF_ALREADY_EXIST.toString(), vfModuleName, cloudOwner,
-                        cloudSiteId, tenantId, OPENSTACK, QUERY_STACK, ErrorCode.DataError.getValue(),
-                        STACK + " " + vfModuleName + " already exists and is " + "in FAILED state");
-                logger.debug(error);
-                throw new VnfAlreadyExists(vfModuleName, cloudOwner, cloudSiteId, tenantId,
-                        heatStack.getCanonicalName());
-            }
-            if (status == HeatStatus.UNKNOWN || status == HeatStatus.UPDATED) {
-                // fail - it exists and is in a FAILED state
-                String error = CREATE_VF_STACK + " " + vfModuleName + " already exists and has status "
-                        + status.toString() + " in " + cloudOwner + "/" + cloudSiteId + "/" + tenantId
-                        + "; requires manual intervention.";
-                logger.error(LoggingAnchor.NINE, MessageEnum.RA_VNF_ALREADY_EXIST.toString(), vfModuleName, cloudOwner,
-                        cloudSiteId, tenantId, OPENSTACK, QUERY_STACK, ErrorCode.DataError.getValue(),
-                        STACK + " " + vfModuleName + " already exists and is " + "in UPDATED or UNKNOWN state");
-                logger.debug(error);
-                throw new VnfAlreadyExists(vfModuleName, cloudOwner, cloudSiteId, tenantId,
-                        heatStack.getCanonicalName());
-            }
-            if (status == HeatStatus.CREATED) {
-                // fail - it exists
-                if (failIfExists != null && failIfExists) {
-                    String error = CREATE_VF_STACK + " " + vfModuleName + " already exists in " + cloudOwner + "/"
-                            + cloudSiteId + "/" + tenantId;
-                    logger.error(LoggingAnchor.NINE, MessageEnum.RA_VNF_ALREADY_EXIST.toString(), vfModuleName,
-                            cloudOwner, cloudSiteId, tenantId, OPENSTACK, QUERY_STACK, ErrorCode.DataError.getValue(),
-                            STACK + " " + vfModuleName + " already exists");
-                    logger.debug(error);
-                    throw new VnfAlreadyExists(vfModuleName, cloudOwner, cloudSiteId, tenantId,
-                            heatStack.getCanonicalName());
-                } else {
-                    logger.debug("Found Existing stack, status={}", heatStack.getStatus());
-                    // Populate the outputs from the existing stack.
-                    vnfId.value = heatStack.getCanonicalName();
-                    outputs.value = copyStringOutputs(heatStack.getOutputs());
-                    rollback.value = vfRollback; // Default rollback - no updates performed
-                }
-            }
-            return;
-
-        }
-
         // handle a nestedStackId if sent- this one would be for the volume - so applies to both Vf and Vnf
         StackInfo nestedHeatStack = null;
-
         Map<String, Object> nestedVolumeOutputs = null;
         if (nestedStackId != null) {
             try {
                 logger.debug("Querying for nestedStackId = {}", nestedStackId);
-                nestedHeatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, nestedStackId);
+                nestedHeatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, nestedStackId);
             } catch (MsoException me) {
                 // Failed to query the Stack due to an openstack exception.
                 // Convert to a generic VnfException
@@ -761,7 +677,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         if (nestedBaseStackId != null) {
             try {
                 logger.debug("Querying for nestedBaseStackId = {}", nestedBaseStackId);
-                nestedBaseHeatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, nestedBaseStackId);
+                nestedBaseHeatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, nestedBaseStackId);
             } catch (MsoException me) {
                 // Failed to query the Stack due to an openstack exception.
                 // Convert to a generic VnfException
@@ -790,7 +706,6 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         }
 
         try {
-            // Retrieve the VF
             VfModule vf = null;
             VnfResource vnfResource = null;
             VfModuleCustomization vfmc = null;
@@ -985,7 +900,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
                 logger.debug("Contents of nestedTemplates - to be added to files: on stack:");
                 for (HeatTemplate entry : nestedTemplates) {
                     nestedTemplatesChecked.put(entry.getTemplateName(), entry.getTemplateBody());
-                    logger.debug(entry.getTemplateName() + " -> " + entry.getTemplateBody());
+                    logger.debug("Adding Nested Template", entry.getTemplateName());
                 }
             } else {
                 logger.debug("No nested templates found - nothing to do here");
@@ -1107,13 +1022,13 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
             // Step 1 - convert what we got as inputs (Map<String, String>) to a
             // Map<String, Object> - where the object matches the param type identified in the template
             // This will also not copy over params that aren't identified in the template
-            goldenInputs = heat.convertInputMap(inputs, heatTemplate);
+            goldenInputs = msoHeatUtils.convertInputMap(inputs, heatTemplate);
             // Step 2 - now simply add the outputs as we received them - no need to convert to string
             logger.debug("Now add in the base stack outputs if applicable");
-            heat.copyBaseOutputsToInputs(goldenInputs, baseStackOutputs, parameterNames, aliasToParam);
+            msoHeatUtils.copyBaseOutputsToInputs(goldenInputs, baseStackOutputs, parameterNames, aliasToParam);
             // Step 3 - add the volume inputs if any
             logger.debug("Now add in the volume stack outputs if applicable");
-            heat.copyBaseOutputsToInputs(goldenInputs, nestedVolumeOutputs, parameterNames, aliasToParam);
+            msoHeatUtils.copyBaseOutputsToInputs(goldenInputs, nestedVolumeOutputs, parameterNames, aliasToParam);
 
             for (HeatTemplateParam parm : heatTemplate.getParameters()) {
                 logger.debug("Parameter:'" + parm.getParamName() + "', isRequired=" + parm.isRequired() + ", alias="
@@ -1186,15 +1101,16 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
             // Have the tenant. Now deploy the stack itself
             // Ignore MsoTenantNotFound and MsoStackAlreadyExists exceptions
             // because we already checked for those.
+
+            StackInfo heatStack = null;
             try {
                 if (backout == null) {
                     backout = true;
                 }
-                if (heat != null) {
-
-                    heatStack = heat.createStack(cloudSiteId, cloudOwner, tenantId, vfModuleName, null, template,
-                            goldenInputs, true, heatTemplate.getTimeoutMinutes(), newEnvironmentString,
-                            nestedTemplatesChecked, heatFilesObjects, backout.booleanValue());
+                if (msoHeatUtils != null) {
+                    heatStack = msoHeatUtils.createStack(cloudSiteId, cloudOwner, tenantId, vfModuleName, null,
+                            template, goldenInputs, true, heatTemplate.getTimeoutMinutes(), newEnvironmentString,
+                            nestedTemplatesChecked, heatFilesObjects, backout.booleanValue(), failIfExists);
                 } else {
                     throw new MsoHeatNotFoundException();
                 }
@@ -1251,15 +1167,9 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
     @Override
     public void deleteVfModule(String cloudSiteId, String cloudOwner, String tenantId, String vnfName,
             MsoRequest msoRequest, Holder<Map<String, String>> outputs) throws VnfException {
-
-        logger.debug("Deleting VF {} in ", vnfName, cloudOwner + "/" + cloudSiteId + "/" + tenantId);
-        // Will capture execution time for metrics
-
-        // 1702 capture the output parameters on a delete
-        // so we'll need to query first
         Map<String, Object> stackOutputs;
         try {
-            stackOutputs = heat.queryStackForOutputs(cloudSiteId, cloudOwner, tenantId, vnfName);
+            stackOutputs = msoHeatUtils.queryStackForOutputs(cloudSiteId, cloudOwner, tenantId, vnfName);
         } catch (MsoException me) {
             // Failed to query the Stack due to an openstack exception.
             // Convert to a generic VnfException
@@ -1286,7 +1196,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         }
 
         try {
-            heat.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfName, true);
+            msoHeatUtils.deleteStack(tenantId, cloudOwner, cloudSiteId, vnfName, true, 118);
         } catch (MsoException me) {
             me.addContext(DELETE_VNF);
             // Failed to query the Stack due to an openstack exception.
@@ -1321,9 +1231,6 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
                 logger.error("Exception encountered while sending Confirm to Valet ", e);
             }
         }
-
-        // On success, nothing is returned.
-        return;
     }
 
     @Override
@@ -1417,7 +1324,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         StackInfo heatStack;
         logger.debug("UpdateVfModule - querying for {}", vfModuleName);
         try {
-            heatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, vfModuleName);
+            heatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, vfModuleName);
         } catch (MsoException me) {
             // Failed to query the Stack due to an openstack exception.
             // Convert to a generic VnfException
@@ -1452,7 +1359,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         if (nestedStackId != null) {
             try {
                 logger.debug("Querying for nestedStackId = {}", nestedStackId);
-                nestedHeatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, nestedStackId);
+                nestedHeatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, nestedStackId);
             } catch (MsoException me) {
                 // Failed to query the Stack due to an openstack exception.
                 // Convert to a generic VnfException
@@ -1475,7 +1382,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
             } else {
                 logger.debug("Found nested heat stack - copying values to inputs *later*");
                 nestedVolumeOutputs = nestedHeatStack.getOutputs();
-                heat.copyStringOutputsToInputs(inputs, nestedHeatStack.getOutputs(), false);
+                msoHeatUtils.copyStringOutputsToInputs(inputs, nestedHeatStack.getOutputs(), false);
             }
         }
         // handle a nestedBaseStackId if sent - this is the stack ID of the base.
@@ -1484,7 +1391,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         if (nestedBaseStackId != null) {
             try {
                 logger.debug("Querying for nestedBaseStackId = {}", nestedBaseStackId);
-                nestedBaseHeatStack = heat.queryStack(cloudSiteId, cloudOwner, tenantId, nestedBaseStackId);
+                nestedBaseHeatStack = msoHeatUtils.queryStack(cloudSiteId, cloudOwner, tenantId, nestedBaseStackId);
             } catch (MsoException me) {
                 // Failed to query the Stack due to an openstack exception.
                 // Convert to a generic VnfException
@@ -1507,14 +1414,10 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
             } else {
                 logger.debug("Found nested base heat stack - copying values to inputs *later*");
                 baseStackOutputs = nestedBaseHeatStack.getOutputs();
-                heat.copyStringOutputsToInputs(inputs, nestedBaseHeatStack.getOutputs(), false);
+                msoHeatUtils.copyStringOutputsToInputs(inputs, nestedBaseHeatStack.getOutputs(), false);
             }
         }
 
-        // Ready to deploy the new VNF
-
-
-
         // Retrieve the VF definition
         VnfResource vnfResource = null;
         VfModule vf = null;
@@ -1924,9 +1827,8 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
                 parsedVfModuleName = "unknown";
             }
             sendResponseToValet = this.valetUpdateRequest(cloudSiteId, cloudOwner, tenantId, heatFilesObjects,
-                    nestedTemplatesChecked, parsedVfModuleName, false, heatTemplate, newEnvironmentString,
-                    (HashMap<String, Object>) goldenInputs, msoRequest, inputs, failRequestOnValetFailure,
-                    valetModifiedParamsHolder);
+                    nestedTemplatesChecked, parsedVfModuleName, false, heatTemplate, newEnvironmentString, goldenInputs,
+                    msoRequest, inputs, failRequestOnValetFailure, valetModifiedParamsHolder);
             if (sendResponseToValet) {
                 goldenInputs = valetModifiedParamsHolder.value;
             }
@@ -2064,7 +1966,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         boolean valetSucceeded = false;
         String valetErrorMessage = "more detail not available";
         try {
-            String keystoneUrl = heat.getCloudSiteKeystoneUrl(cloudSiteId);
+            String keystoneUrl = msoHeatUtils.getCloudSiteKeystoneUrl(cloudSiteId);
             Map<String, Object> files =
                     this.combineGetFilesAndNestedTemplates(heatFilesObjects, nestedTemplatesChecked);
             HeatRequest heatRequest = new HeatRequest(vfModuleName, backout, heatTemplate.getTimeoutMinutes(),
@@ -2126,7 +2028,7 @@ public class MsoVnfAdapterImpl implements MsoVnfAdapter {
         boolean valetSucceeded = false;
         String valetErrorMessage = "more detail not available";
         try {
-            String keystoneUrl = heat.getCloudSiteKeystoneUrl(cloudSiteId);
+            String keystoneUrl = msoHeatUtils.getCloudSiteKeystoneUrl(cloudSiteId);
             Map<String, Object> files =
                     this.combineGetFilesAndNestedTemplates(heatFilesObjects, nestedTemplatesChecked);
             HeatRequest heatRequest = new HeatRequest(vfModuleName, false, heatTemplate.getTimeoutMinutes(),
diff --git a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/VolumeAdapterRestV2.java b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/VolumeAdapterRestV2.java
index 77c9b1e2f5..89eb6a161d 100644
--- a/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/VolumeAdapterRestV2.java
+++ b/adapters/mso-openstack-adapters/src/main/java/org/onap/so/adapters/vnf/VolumeAdapterRestV2.java
@@ -105,7 +105,6 @@ public class VolumeAdapterRestV2 {
             @ApiResponse(code = 500, message = "create vnfVolume failed, examine entity object for details")})
     public Response createVNFVolumes(@ApiParam(value = "mode", required = true) @QueryParam("mode") String mode,
             @ApiParam(value = "CreateVolumeGroupRequest", required = true) final CreateVolumeGroupRequest req) {
-        logger.debug("createVNFVolumes enter: {}", req.toJsonString());
         CreateVNFVolumesTask task = new CreateVNFVolumesTask(req, mode);
         if (req.isSynchronous()) {
             // This is a synchronous request
diff --git a/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/network/NetworkAdapterRestTest.java b/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/network/NetworkAdapterRestTest.java
index 1f4c205e7a..412abeab3b 100644
--- a/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/network/NetworkAdapterRestTest.java
+++ b/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/network/NetworkAdapterRestTest.java
@@ -194,9 +194,7 @@ public class NetworkAdapterRestTest extends BaseRestTestUtils {
     public void testRollbackNetwork() throws IOException {
 
         RollbackNetworkRequest request = new RollbackNetworkRequest();
-
         MsoRequest msoReq = new MsoRequest();
-
         msoReq.setRequestId(MSO_REQUEST_ID);
         msoReq.setServiceInstanceId(MSO_SERVICE_INSTANCE_ID);
 
diff --git a/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/vnf/MsoVnfAdapterImplTest.java b/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/vnf/MsoVnfAdapterImplTest.java
index 651e2c9a73..0d972bb4d2 100644
--- a/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/vnf/MsoVnfAdapterImplTest.java
+++ b/adapters/mso-openstack-adapters/src/test/java/org/onap/so/adapters/vnf/MsoVnfAdapterImplTest.java
@@ -40,7 +40,6 @@ import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-import org.onap.so.adapters.vnf.exceptions.VnfAlreadyExists;
 import org.onap.so.adapters.vnf.exceptions.VnfException;
 import org.onap.so.adapters.vnf.exceptions.VnfNotFound;
 import org.onap.so.db.catalog.beans.HeatEnvironment;
@@ -50,15 +49,15 @@ import org.onap.so.db.catalog.beans.VfModule;
 import org.onap.so.db.catalog.beans.VfModuleCustomization;
 import org.onap.so.db.catalog.beans.VnfResource;
 import org.onap.so.entity.MsoRequest;
-import org.onap.so.openstack.beans.HeatStatus;
-import org.onap.so.openstack.beans.StackInfo;
 import org.onap.so.openstack.beans.VnfRollback;
 import org.onap.so.openstack.exceptions.MsoException;
 import org.onap.so.openstack.utils.MsoHeatUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import com.github.tomakehurst.wiremock.stubbing.Scenario;
 
 
 public class MsoVnfAdapterImplTest extends BaseRestTestUtils {
+
     @Rule
     public ExpectedException expectedException = ExpectedException.none();
 
@@ -71,17 +70,7 @@ public class MsoVnfAdapterImplTest extends BaseRestTestUtils {
     String vnfName = "DEV-VF-1802-it3-pwt3-v6-vSAMP10a-addon2-Replace-1001/stackId";
 
     @Test
-    @Ignore
-    public void healthCheckVNFTest() {
-        MsoVnfAdapterImpl instance = new MsoVnfAdapterImpl();
-        instance.healthCheck();
-    }
-
-    @Test
     public void createVnfTest() throws Exception {
-        StackInfo info = new StackInfo();
-        info.setStatus(HeatStatus.CREATED);
-
         mockOpenStackResponseAccess(wireMockServer, wireMockPort);
         mockOpenStackGetStackVfModule_200(wireMockServer);
 
@@ -90,105 +79,57 @@ public class MsoVnfAdapterImplTest extends BaseRestTestUtils {
         Map<String, Object> map = new HashMap<>();
         map.put("key1", "value1");
         instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.FALSE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
+                "VFMOD", null, null, "b4ea86b4-253f-11e7-93ae-92361f002671", map, Boolean.TRUE, Boolean.TRUE,
+                Boolean.FALSE, msoRequest, new Holder<>(), new Holder<Map<String, String>>(),
+                new Holder<VnfRollback>());
     }
 
     @Test
-    public void createVnfTest_HeatStatusUpdating() throws Exception {
-        expectedException.expect(VnfAlreadyExists.class);
+    public void createVnfTest_HeatStatusFailed() throws Exception {
         mockOpenStackResponseAccess(wireMockServer, wireMockPort);
-
         wireMockServer.stubFor(get(
                 urlPathEqualTo("/mockPublicUrl/stacks/DEV-VF-1802-it3-pwt3-v6-vSAMP10a-addon2-Replace-1001/stackId"))
                         .willReturn(aResponse().withHeader("Content-Type", "application/json")
-                                .withBodyFile("OpenstackResponse_Stack_Updating_VfModule.json")
-                                .withStatus(HttpStatus.SC_OK)));
-
-        MsoRequest msoRequest = getMsoRequest();
-
-        Map<String, Object> map = new HashMap<>();
-        map.put("key1", "value1");
-        instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.FALSE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
-    }
-
-    @Test
-    public void createVnfTest_HeatStatusUpdated() throws Exception {
-        expectedException.expect(VnfAlreadyExists.class);
-        mockOpenStackResponseAccess(wireMockServer, wireMockPort);
+                                .withBodyFile("OpenstackResponse_Stack_Failed_VfModule.json")
+                                .withStatus(HttpStatus.SC_OK))
+                        .inScenario("HeatStatusFailure").whenScenarioStateIs(Scenario.STARTED)
+                        .willSetStateTo("HeatStackFailed"));
 
         wireMockServer.stubFor(get(
                 urlPathEqualTo("/mockPublicUrl/stacks/DEV-VF-1802-it3-pwt3-v6-vSAMP10a-addon2-Replace-1001/stackId"))
                         .willReturn(aResponse().withHeader("Content-Type", "application/json")
-                                .withBodyFile("OpenstackResponse_StackId.json").withStatus(HttpStatus.SC_OK)));
+                                .withBodyFile("OpenstackResponse_Stack_Created_VfModule.json")
+                                .withStatus(HttpStatus.SC_OK))
+                        .inScenario("HeatStatusFailure").whenScenarioStateIs("HeatStackFailed")
+                        .willSetStateTo("HeatStackSuccess"));
 
         MsoRequest msoRequest = getMsoRequest();
 
         Map<String, Object> map = new HashMap<>();
         map.put("key1", "value1");
         instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.FALSE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
+                "VFMOD", null, null, "b4ea86b4-253f-11e7-93ae-92361f002671", map, Boolean.FALSE, Boolean.TRUE,
+                Boolean.FALSE, msoRequest, new Holder<>(), new Holder<Map<String, String>>(),
+                new Holder<VnfRollback>());
     }
 
-    @Test
-    public void createVnfTest_HeatStatusFailed() throws Exception {
-        expectedException.expect(VnfAlreadyExists.class);
-        mockOpenStackResponseAccess(wireMockServer, wireMockPort);
-
-        wireMockServer.stubFor(get(
-                urlPathEqualTo("/mockPublicUrl/stacks/DEV-VF-1802-it3-pwt3-v6-vSAMP10a-addon2-Replace-1001/stackId"))
-                        .willReturn(aResponse().withHeader("Content-Type", "application/json")
-                                .withBodyFile("OpenstackResponse_Stack_Failed_VfModule.json")
-                                .withStatus(HttpStatus.SC_OK)));
-
-        MsoRequest msoRequest = getMsoRequest();
 
-        Map<String, Object> map = new HashMap<>();
-        map.put("key1", "value1");
-        instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.FALSE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
-    }
 
     @Test
     public void createVnfTest_HeatStatusCreated() throws Exception {
-        expectedException.expect(VnfAlreadyExists.class);
         mockOpenStackResponseAccess(wireMockServer, wireMockPort);
-
         mockOpenStackGetStackVfModule_200(wireMockServer);
-
         MsoRequest msoRequest = getMsoRequest();
-
         Map<String, Object> map = new HashMap<>();
         map.put("key1", "value1");
         instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.TRUE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
+                "VFMOD", null, null, "b4ea86b4-253f-11e7-93ae-92361f002671", map, Boolean.TRUE, Boolean.TRUE,
+                Boolean.FALSE, msoRequest, new Holder<>(), new Holder<Map<String, String>>(),
+                new Holder<VnfRollback>());
     }
 
 
     @Test
-    public void createVnfTest_ExceptionInGettingHeat() throws Exception {
-        expectedException.expect(VnfException.class);
-        MsoRequest msoRequest = getMsoRequest();
-
-        Map<String, Object> map = new HashMap<>();
-        map.put("key1", "value1");
-        instance.createVfModule("mtn13", "CloudOwner", "88a6ca3ee0394ade9403f075db23167e", "vnf", "1", "", vnfName, "",
-                "VFMOD", "volumeGroupHeatStackId|1", "baseVfHeatStackId", "88a6ca3ee0394ade9403f075db23167e", map,
-                Boolean.FALSE, Boolean.TRUE, Boolean.FALSE, msoRequest, new Holder<>(),
-                new Holder<Map<String, String>>(), new Holder<VnfRollback>());
-    }
-
-    @Test
     public void createVnfTest_NestedHeatStatusNotFound() throws Exception {
         expectedException.expect(VnfException.class);
         mockOpenStackResponseAccess(wireMockServer, wireMockPort);
diff --git a/adapters/mso-openstack-adapters/src/test/resources/application-test.yaml b/adapters/mso-openstack-adapters/src/test/resources/application-test.yaml
index 7cc7697129..20fd812ab9 100644
--- a/adapters/mso-openstack-adapters/src/test/resources/application-test.yaml
+++ b/adapters/mso-openstack-adapters/src/test/resources/application-test.yaml
@@ -1,5 +1,3 @@
-# will be used as entry in DB to say SITE OFF/ON for healthcheck
-# MSO Properties go here
 org:
   onap:
     so:
@@ -41,7 +39,7 @@ org:
           default_keystone_reg_ex: "/[vV][0-9]"
           default_tenant_description: ECOMP Tenant
           default_region_type: single
-          default_user_role: admin
+          default_user_role: adminev
           default_success_status_string: Success
           default_no_regions_status_string: no regions
           default_orm_request_path: /v1/orm/customers/