Prevent race conditions on same distributionId.

- Code to include optimistic lock to prevent race conditions on same
distributionId.

Change-Id: Ibe110b32f2472d991a4a3e03e3d15d5e4deefd65
Issue-ID: SO-1566
Signed-off-by: Benjamin, Max (mb388a) <mb388a@us.att.com>

4 files changed, 38 insertions, 19 deletions

diff --git a/asdc-controller/src/main/java/org/onap/so/asdc/client/ASDCController.java b/asdc-controller/src/main/java/org/onap/so/asdc/client/ASDCController.java
index 7a02f47d3f..a21deee51a 100644
--- a/asdc-controller/src/main/java/org/onap/so/asdc/client/ASDCController.java
+++ b/asdc-controller/src/main/java/org/onap/so/asdc/client/ASDCController.java
@@ -30,6 +30,7 @@ import java.nio.file.Paths;
 import java.util.List;
 import java.util.Optional;
 
+import org.hibernate.StaleObjectStateException;
 import org.onap.sdc.api.IDistributionClient;
 import org.onap.sdc.api.consumer.IDistributionStatusMessage;
 import org.onap.sdc.api.consumer.IFinalDistrStatusMessage;
@@ -58,6 +59,7 @@ import org.onap.so.db.request.data.repository.WatchdogDistributionStatusReposito
 import org.onap.so.logger.MessageEnum;
 import org.onap.so.logger.MsoLogger;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.orm.ObjectOptimisticLockingFailureException;
 import org.springframework.stereotype.Component;
 
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
@@ -592,6 +594,7 @@ public class ASDCController {
         try {
         	LOGGER.debug(ASDCNotificationLogging.dumpASDCNotification(iNotif));
 			LOGGER.info(MessageEnum.ASDC_RECEIVE_SERVICE_NOTIF, iNotif.getServiceUUID(), "ASDC", "treatNotification");
+						
 			this.changeControllerStatus(ASDCControllerStatus.BUSY);
 			Optional<String> notificationMessage = getNotificationJson(iNotif);
 			toscaInstaller.processWatchdog(iNotif.getDistributionID(), iNotif.getServiceUUID(), notificationMessage,
@@ -619,7 +622,7 @@ public class ASDCController {
     				distributionStatus = wd.getOverallDistributionStatus(iNotif.getDistributionID());
     				Thread.sleep(watchDogTimeout / 10);    		
     			}catch(Exception e){
-    				LOGGER.debug ("Exception in Watchdog Loop " + e);
+    				LOGGER.debug ("Exception in Watchdog Loop " + e.getMessage());
     				Thread.sleep(watchDogTimeout / 10);
     			}
     			
@@ -651,7 +654,7 @@ public class ASDCController {
         		LOGGER.debug ("A&AI Updated succefully with Distribution Status!");
         	}
         	catch(Exception e) {
-        		LOGGER.debug ("Exception in Watchdog executePatchAAI(): " + e);
+        		LOGGER.debug ("Exception in Watchdog executePatchAAI(): " + e.getMessage());
         		watchdogError = "Error calling A&AI " + e.getMessage();
         		if(e.getCause() != null) {
         			LOGGER.debug ("Exception caused by: " + e.getCause().getMessage());
@@ -671,8 +674,14 @@ public class ASDCController {
         		wdsRepo.save(wds);
         	}
         	
-        	
 
+        } catch(ObjectOptimisticLockingFailureException e) {
+        	
+        	LOGGER.debug ("OptimisticLockingFailure for DistributionId: " + iNotif.getDistributionID() + " Another process has already altered this distribution, so not going to process it on this site.");
+        	LOGGER.error (MessageEnum.ASDC_GENERAL_EXCEPTION_ARG,
+                    "Database concurrency exception: ",  "ASDC", "treatNotification", MsoLogger.ErrorCode.BusinessProcesssError, "RuntimeException in treatNotification",
+                    e);
+        	
         } catch (Exception e) {
             LOGGER.error (MessageEnum.ASDC_GENERAL_EXCEPTION_ARG,
                           "Unexpected exception caught during the notification processing",  "ASDC", "treatNotification", MsoLogger.ErrorCode.SchemaError, "RuntimeException in treatNotification",
diff --git a/asdc-controller/src/main/java/org/onap/so/asdc/installer/heat/ToscaResourceInstaller.java b/asdc-controller/src/main/java/org/onap/so/asdc/installer/heat/ToscaResourceInstaller.java
index 10b8ba7c16..60d5d7e061 100644
--- a/asdc-controller/src/main/java/org/onap/so/asdc/installer/heat/ToscaResourceInstaller.java
+++ b/asdc-controller/src/main/java/org/onap/so/asdc/installer/heat/ToscaResourceInstaller.java
@@ -34,6 +34,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.hibernate.StaleObjectStateException;
 import org.hibernate.exception.ConstraintViolationException;
 import org.hibernate.exception.LockAcquisitionException;
 import org.onap.sdc.api.notification.IArtifactInfo;
@@ -124,6 +125,7 @@ import org.onap.so.db.request.data.repository.WatchdogServiceModVerIdLookupRepos
 import org.onap.so.logger.MessageEnum;
 import org.onap.so.logger.MsoLogger;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.orm.ObjectOptimisticLockingFailureException;
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -701,10 +703,17 @@ public class ToscaResourceInstaller {
 				distributionNotification, consumerId);
 		watchdogModVerIdLookupRepository.saveAndFlush(modVerIdLookup);
 		
-		WatchdogDistributionStatus distributionStatus = new WatchdogDistributionStatus(distributionId);
-		watchdogDistributionStatusRepository.saveAndFlush(distributionStatus);
+		try{
+		
+			WatchdogDistributionStatus distributionStatus = new WatchdogDistributionStatus(distributionId);
+			watchdogDistributionStatusRepository.saveAndFlush(distributionStatus);
+			
+		} catch(ObjectOptimisticLockingFailureException e){
+			logger.debug("ObjectOptimisticLockingFailureException in processWatchdog : " + e.toString());
+			throw e;
+		}
 	}
-
+	
 	protected void extractHeatInformation(ToscaResourceStructure toscaResourceStruct,
 			VfResourceStructure vfResourceStructure) {
 		for (VfModuleArtifact vfModuleArtifact : vfResourceStructure.getArtifactsMapByUUID().values()) {
diff --git a/asdc-controller/src/test/resources/data.sql b/asdc-controller/src/test/resources/data.sql
index 70737abf5a..7cd972ca63 100644
--- a/asdc-controller/src/test/resources/data.sql
+++ b/asdc-controller/src/test/resources/data.sql
@@ -24,19 +24,19 @@ INSERT INTO temp_network_heat_template_lookup(NETWORK_RESOURCE_MODEL_NAME, HEAT_
 
 
 --------START Request DB INSERTS --------
-insert into requestdb.watchdog_distributionid_status(DISTRIBUTION_ID, DISTRIBUTION_ID_STATUS) values 
-('watchdogTestStatusSuccess', 'SUCCESS'),
-('watchdogTestStatusFailure', 'FAILURE'),
-('watchdogTestStatusTimeout', 'TIMEOUT'),
-('watchdogTestStatusIncomplete', 'INCOMPLETE'),
-('watchdogTestStatusException', 'EXCEPTION'),
-('watchdogTestStatusNull', 'NULL'),
-('testStatusSuccessTosca', 'SUCCESS'),
-('testStatusFailureTosca', 'FAILURE'),
-('testStatusTimeoutTosca', 'TIMEOUT'),
-('testStatusIncompleteTosca', 'INCOMPLETE'),
-('testStatusExceptionTosca', 'EXCEPTION'),
-('testStatusNullTosca', 'NULL');
+insert into requestdb.watchdog_distributionid_status(DISTRIBUTION_ID, DISTRIBUTION_ID_STATUS,LOCK_VERSION) values 
+('watchdogTestStatusSuccess', 'SUCCESS',0),
+('watchdogTestStatusFailure', 'FAILURE',0),
+('watchdogTestStatusTimeout', 'TIMEOUT',0),
+('watchdogTestStatusIncomplete', 'INCOMPLETE',0),
+('watchdogTestStatusException', 'EXCEPTION',0),
+('watchdogTestStatusNull', 'NULL',0),
+('testStatusSuccessTosca', 'SUCCESS',0),
+('testStatusFailureTosca', 'FAILURE',0),
+('testStatusTimeoutTosca', 'TIMEOUT',0),
+('testStatusIncompleteTosca', 'INCOMPLETE',0),
+('testStatusExceptionTosca', 'EXCEPTION',0),
+('testStatusNullTosca', 'NULL',0);
 
 --WatchdogDistrubutionTest
 insert into requestdb.watchdog_per_component_distribution_status(DISTRIBUTION_ID, COMPONENT_NAME, COMPONENT_DISTRIBUTION_STATUS) values
diff --git a/asdc-controller/src/test/resources/schema.sql b/asdc-controller/src/test/resources/schema.sql
index 010b36dcf1..43572c476d 100644
--- a/asdc-controller/src/test/resources/schema.sql
+++ b/asdc-controller/src/test/resources/schema.sql
@@ -990,6 +990,7 @@ CREATE TABLE `site_status` (
 CREATE TABLE `watchdog_distributionid_status` (
   `DISTRIBUTION_ID` varchar(45) NOT NULL,
   `DISTRIBUTION_ID_STATUS` varchar(45) DEFAULT NULL,
+  `LOCK_VERSION` int NOT NULL,
   `CREATE_TIME` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `MODIFY_TIME` datetime DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`DISTRIBUTION_ID`)