Support updated data types in service import

Signed-off-by: MichaelMorris <michael.morris@est.tech>
Issue-ID: SDC-4140
Change-Id: Ib66d47a0f566b648722ce86cfc4e208880551a29

10 files changed, 138 insertions, 42 deletions

diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/DataTypeDefinition.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/DataTypeDefinition.java
index bf9f4d79a6..f208891e21 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/DataTypeDefinition.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/DataTypeDefinition.java
@@ -50,6 +50,7 @@ public class DataTypeDefinition extends DataTypeDataDefinition {
         this.setConstraints(dataTypeDefinition.getConstraints());
         this.setDescription(dataTypeDefinition.getDescription());
         this.setModel(dataTypeDefinition.getModel());
+        this.setProperties(dataTypeDefinition.getProperties());
     }
 
     public List<PropertyConstraint> safeGetConstraints() {
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java
index 0b4d02a0ab..338bbe15a9 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/cache/ApplicationDataTypeCache.java
@@ -198,6 +198,21 @@ public class ApplicationDataTypeCache implements ApplicationCache<DataTypeDefini
             readWriteLock.readLock().unlock();
         }
     }
+    
+    public void reload(final String model, final String uniqueId) {
+        final Either<DataTypeDefinition, JanusGraphOperationStatus> dataTypeDefEither = propertyOperation.getDataTypeByUid(uniqueId);
+        if (dataTypeDefEither.isLeft()) {
+            DataTypeDefinition dataTypeDef = dataTypeDefEither.left().value();
+            if (getDataTypeDefinitionMapByModel(model).containsKey(dataTypeDef.getName())) {
+                try {
+                    readWriteLock.readLock().lock();
+                    getDataTypeDefinitionMapByModel(model).put(dataTypeDef.getName(), dataTypeDef);
+                } finally {
+                    readWriteLock.readLock().unlock();
+                }
+            }
+        }
+    }
 
     private Map<String, DataTypeDefinition> getDataTypeDefinitionMapByModel(final String model) {
         return dataTypesByModelCacheMap.containsKey(model) ? dataTypesByModelCacheMap.get(model) : new HashMap<>();
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/ModelOperation.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/ModelOperation.java
index 01e5cdcd40..0d462c9874 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/ModelOperation.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/ModelOperation.java
@@ -26,12 +26,16 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -50,6 +54,7 @@ import org.openecomp.sdc.be.data.model.ToscaImportByModel;
 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
 import org.openecomp.sdc.be.datatypes.enums.ModelTypeEnum;
 import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
+import org.openecomp.sdc.be.model.DataTypeDefinition;
 import org.openecomp.sdc.be.model.Model;
 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.exception.ModelOperationExceptionSupplier;
 import org.openecomp.sdc.be.model.jsonjanusgraph.operations.exception.OperationException;
@@ -379,4 +384,50 @@ public class ModelOperation {
         this.modelElementOperation = modelElementOperation;
     }
 
+    @SuppressWarnings("unchecked")
+    public void updateTypesInAdditionalTypesImport(final ElementTypeEnum elementTypeEnum, final String typesYaml, final String modelName) {
+        final Optional<ToscaImportByModel> additionalTypeDefinitionsImportOptional = getAdditionalTypes(modelName);
+
+        if (additionalTypeDefinitionsImportOptional.isPresent()) {
+
+            final Map<String, Object> existingTypeContent = getExistingTypes(elementTypeEnum, additionalTypeDefinitionsImportOptional.get());
+            final Set<String> existingTypeNames = existingTypeContent.keySet();
+
+            final Map<String, Object> typesToUpate = new HashMap<>();
+
+            final Map<String, Object> newTypesYaml = new Yaml().load(typesYaml);
+            newTypesYaml.entrySet().stream().filter(entry -> existingTypeNames.contains(entry.getKey())).forEach(newTypeToUpdate -> {
+
+                final Map<String, Object> propertiesInNewDef = (Map<String, Object>) ((Map<String, Object>) newTypeToUpdate.getValue()).get("properties");
+                final Map<String, Object> existingProperties =
+                        (Map<String, Object>) ((Map<String, Object>) existingTypeContent.get(newTypeToUpdate.getKey())).get("properties");
+
+                final List<Entry<String, Object>> propertiesMissingFromNewDef = MapUtils.isEmpty(existingProperties) ? Collections.emptyList()
+                        : existingProperties.entrySet().stream()
+                                .filter(existingPropEntry -> !propertiesInNewDef.keySet().contains(existingPropEntry.getKey()))
+                                .collect(Collectors.toList());
+
+                if (CollectionUtils.isNotEmpty(propertiesMissingFromNewDef)) {
+                    typesToUpate.put(newTypeToUpdate.getKey(), newTypeToUpdate.getValue());
+
+                    propertiesMissingFromNewDef
+                            .forEach(existingPropToAdd -> propertiesInNewDef.put(existingPropToAdd.getKey(), existingPropToAdd.getValue()));
+                }
+            });
+            if (MapUtils.isNotEmpty(typesToUpate)) {
+                addTypesToDefaultImports(elementTypeEnum, new Yaml().dumpAsMap(typesToUpate), modelName);
+            }
+        }
+    }
+    
+    private  Optional<ToscaImportByModel> getAdditionalTypes(final String modelName) {
+        final List<ToscaImportByModel> modelImportList = toscaModelImportCassandraDao.findAllByModel(modelName);
+        return modelImportList.stream().filter(t -> ADDITIONAL_TYPE_DEFINITIONS_PATH.equals(Path.of(t.getFullPath()))).findAny();
+    }
+    
+    private Map<String, Object> getExistingTypes(final ElementTypeEnum elementTypeEnum, final ToscaImportByModel additionalTypeDefinitionsImport) {
+        final Map<String, Object> existingContent = new Yaml().load(additionalTypeDefinitionsImport.getContent());
+        return  (Map<String, Object>) existingContent.get(elementTypeEnum.getToscaEntryName());
+    }
+
 }
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/PropertyOperation.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/PropertyOperation.java
index 7546e90a82..b5215dc848 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/PropertyOperation.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/operations/impl/PropertyOperation.java
@@ -1822,13 +1822,13 @@ public class PropertyOperation extends AbstractOperation implements IPropertyOpe
             String oldDerivedFromName = oldDataTypeDefinition.getDerivedFromName();
             String dataTypeName = newDataTypeDefinition.getName();
             List<PropertyDefinition> propertiesToAdd = new ArrayList<>();
-            if (isPropertyOmitted(newProperties, oldProperties, dataTypeName) || isPropertyTypeChanged(dataTypeName, newProperties, oldProperties,
-                propertiesToAdd) || isDerivedFromNameChanged(dataTypeName, newDerivedFromName, oldDerivedFromName)) {
+            if (isPropertyTypeChanged(dataTypeName, newProperties, oldProperties, propertiesToAdd)
+                    || isDerivedFromNameChanged(dataTypeName, newDerivedFromName, oldDerivedFromName)) {
                 log.debug("The new data type {} is invalid.", dataTypeName);
                 result = Either.right(StorageOperationStatus.CANNOT_UPDATE_EXISTING_ENTITY);
                 return result;
             }
-            if (propertiesToAdd == null || propertiesToAdd.isEmpty()) {
+            if (CollectionUtils.isEmpty(propertiesToAdd)) {
                 log.debug("No new properties has been defined in the new data type {}", newDataTypeDefinition);
                 result = Either.right(StorageOperationStatus.OK);
                 return result;
@@ -1935,44 +1935,6 @@ public class PropertyOperation extends AbstractOperation implements IPropertyOpe
         return entryType;
     }
 
-    private boolean isPropertyOmitted(List<PropertyDefinition> newProperties, List<PropertyDefinition> oldProperties, String dataTypeName) {
-        boolean isValid = validateChangeInCaseOfEmptyProperties(newProperties, oldProperties, dataTypeName);
-        if (!isValid) {
-            log.debug("At least one property is missing in the new data type {}", dataTypeName);
-            return false;
-        }
-        if (newProperties != null && oldProperties != null) {
-            List<String> newProps = newProperties.stream().map(PropertyDataDefinition::getName).collect(Collectors.toList());
-            List<String> oldProps = oldProperties.stream().map(PropertyDataDefinition::getName).collect(Collectors.toList());
-            if (!newProps.containsAll(oldProps)) {
-                StringJoiner joiner = new StringJoiner(",", "[", "]");
-                newProps.forEach(joiner::add);
-                log.debug("Properties {} in data type {} are missing, but they already defined in the existing data type", joiner.toString(),
-                    dataTypeName);
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private boolean validateChangeInCaseOfEmptyProperties(List<PropertyDefinition> newProperties, List<PropertyDefinition> oldProperties,
-                                                          String dataTypeName) {
-        if (newProperties != null) {
-            if (newProperties.isEmpty()) {
-                newProperties = null;
-            }
-        }
-        if (oldProperties != null) {
-            if (oldProperties.isEmpty()) {
-                oldProperties = null;
-            }
-        }
-        if ((newProperties == null && oldProperties == null) || (newProperties != null && oldProperties != null)) {
-            return true;
-        }
-        return false;
-    }
-
     private boolean isDerivedFromNameChanged(String dataTypeName, String newDerivedFromName, String oldDerivedFromName) {
         if (newDerivedFromName != null) {
             boolean isEqual = newDerivedFromName.equals(oldDerivedFromName);
diff --git a/catalog-model/src/test/java/org/openecomp/sdc/be/model/operations/impl/ModelOperationTest.java b/catalog-model/src/test/java/org/openecomp/sdc/be/model/operations/impl/ModelOperationTest.java
index 42f52982a8..acff2a3aa2 100644
--- a/catalog-model/src/test/java/org/openecomp/sdc/be/model/operations/impl/ModelOperationTest.java
+++ b/catalog-model/src/test/java/org/openecomp/sdc/be/model/operations/impl/ModelOperationTest.java
@@ -31,6 +31,7 @@ import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.openecomp.sdc.be.model.operations.impl.ModelOperation.ADDITIONAL_TYPE_DEFINITIONS_PATH;
@@ -503,7 +504,28 @@ class ModelOperationTest extends ModelTestBase {
         final ToscaImportByModel actualImport1 = actualImportList.stream().filter(expectedImport1::equals).findFirst().orElse(null);
         assertNotNull(actualImport1);
         assertEquals(expectedImport1.getContent(), actualImport1.getContent());
-
+        
+        // Update the added additional type
+        final var updatedDataTypesPath = testResourcePath.resolve(Path.of("input-data_types-updated.yaml"));
+        final var updatedDataTypes = Files.readString(updatedDataTypesPath);
+        modelOperation.updateTypesInAdditionalTypesImport(ElementTypeEnum.DATA_TYPE, updatedDataTypes, modelName);
+        
+        ArgumentCaptor<List<ToscaImportByModel>> updatedImportListArgumentCaptor = ArgumentCaptor.forClass(List.class);
+        verify(toscaModelImportCassandraDao, times(2)).saveAll(eq(modelName), updatedImportListArgumentCaptor.capture());
+
+        final List<ToscaImportByModel> updatedActualImportList = updatedImportListArgumentCaptor.getValue();
+        assertEquals(2, updatedActualImportList.size());
+        
+        var expectedUpdatedAdditionalTypesImport = new ToscaImportByModel();
+        expectedUpdatedAdditionalTypesImport.setModelId(modelName);
+        expectedUpdatedAdditionalTypesImport.setFullPath(ADDITIONAL_TYPE_DEFINITIONS_PATH.toString());
+        expectedUpdatedAdditionalTypesImport.setContent(Files.readString(testResourcePath.resolve(Path.of("expected-additional_types-2-updated.yaml"))));
+        final ToscaImportByModel actualUpdatedAdditionalTypesImport =
+            actualImportList.stream().filter(expectedUpdatedAdditionalTypesImport::equals).findFirst().orElse(null);
+        assertNotNull(actualUpdatedAdditionalTypesImport);
+        
+        assertTrue(actualUpdatedAdditionalTypesImport.getContent().contains("added_property_1"));
+        assertTrue(actualUpdatedAdditionalTypesImport.getContent().contains("added_property_2"));
     }
 
     @Test
diff --git a/catalog-model/src/test/resources/modelOperation/expected-additional_types-1.yaml b/catalog-model/src/test/resources/modelOperation/expected-additional_types-1.yaml
index c9e6741993..50d9babb60 100644
--- a/catalog-model/src/test/resources/modelOperation/expected-additional_types-1.yaml
+++ b/catalog-model/src/test/resources/modelOperation/expected-additional_types-1.yaml
@@ -5,6 +5,9 @@ data_types:
   tosca.datatypes.nfv.ServiceAvailability:
     derived_from: tosca.datatypes.Root
     description: additional type
+    properties:
+      added_property_1:
+        type: string
   tosca.datatypes.nfv.L2AddressData:
     derived_from: tosca.datatypes.Root
     description: additional type
diff --git a/catalog-model/src/test/resources/modelOperation/expected-additional_types-2-updated.yaml b/catalog-model/src/test/resources/modelOperation/expected-additional_types-2-updated.yaml
new file mode 100644
index 0000000000..ad3d0bfdd8
--- /dev/null
+++ b/catalog-model/src/test/resources/modelOperation/expected-additional_types-2-updated.yaml
@@ -0,0 +1,24 @@
+tosca_definitions_version: tosca_simple_yaml_1_3
+description: Auto-generated file that contains package custom types or types added
+  after system installation.
+data_types:
+  tosca.datatypes.nfv.PreviouslyExistingType1:
+    derived_from: tosca.datatypes.Root
+    description: additional type
+  tosca.datatypes.nfv.PreviouslyExistingType2:
+    derived_from: tosca.datatypes.Root
+    description: additional type
+  tosca.datatypes.nfv.ServiceAvailability:
+    derived_from: tosca.datatypes.Root
+    description: additional type
+    properties:
+      added_property_1:
+        type: string
+      added_property_2:
+        type: string
+  tosca.datatypes.nfv.L2AddressData:
+    derived_from: tosca.datatypes.Root
+    description: additional type
+  tosca.datatypes.nfv.Unknown:
+    derived_from: tosca.datatypes.Root
+    description: additional type
diff --git a/catalog-model/src/test/resources/modelOperation/expected-additional_types-2.yaml b/catalog-model/src/test/resources/modelOperation/expected-additional_types-2.yaml
index 2d2c54206e..a36c828193 100644
--- a/catalog-model/src/test/resources/modelOperation/expected-additional_types-2.yaml
+++ b/catalog-model/src/test/resources/modelOperation/expected-additional_types-2.yaml
@@ -11,6 +11,9 @@ data_types:
   tosca.datatypes.nfv.ServiceAvailability:
     derived_from: tosca.datatypes.Root
     description: additional type
+    properties:
+      added_property_1:
+        type: string
   tosca.datatypes.nfv.L2AddressData:
     derived_from: tosca.datatypes.Root
     description: additional type
diff --git a/catalog-model/src/test/resources/modelOperation/input-data_types-updated.yaml b/catalog-model/src/test/resources/modelOperation/input-data_types-updated.yaml
new file mode 100644
index 0000000000..136585acb6
--- /dev/null
+++ b/catalog-model/src/test/resources/modelOperation/input-data_types-updated.yaml
@@ -0,0 +1,12 @@
+tosca.datatypes.nfv.ServiceAvailability:
+  derived_from: tosca.datatypes.Root
+  description: additional type
+  properties:
+    added_property_2:
+      type: string
+tosca.datatypes.nfv.L2AddressData:
+  derived_from: tosca.datatypes.Root
+  description: additional type
+tosca.datatypes.nfv.Unknown:
+  derived_from: tosca.datatypes.Root
+  description: additional type
\ No newline at end of file
diff --git a/catalog-model/src/test/resources/modelOperation/input-data_types.yaml b/catalog-model/src/test/resources/modelOperation/input-data_types.yaml
index 77b7d977b0..0754631071 100644
--- a/catalog-model/src/test/resources/modelOperation/input-data_types.yaml
+++ b/catalog-model/src/test/resources/modelOperation/input-data_types.yaml
@@ -1,6 +1,9 @@
 tosca.datatypes.nfv.ServiceAvailability:
   derived_from: tosca.datatypes.Root
   description: additional type
+  properties:
+    added_property_1:
+      type: string
 tosca.datatypes.nfv.L2AddressData:
   derived_from: tosca.datatypes.Root
   description: additional type