Update node and data types for SOL001 3.3.1 + CNF enhancements

Types in 2.7.1 folder are types valid for v3.3.1 that have not changed from v2.7.1
Types in 4.1.1 folder are the CNF enhancements which it is assumed will be in v4.1.1

Includes some changes in functionality to support:
1. Deployment of both existing (2.5.1) versions of types and new versions of types through sdc-BE-init
2. Selection of the correct node type definition version at onboarding of an ETSI SOL004 VNF/CNF csar,
based on the declared version supported in the csar (i.e. when a 3.3.1 csar is imported, the node type
definitions valid for 3.3.1 are used (rather than current logic which always selects the latest version)

Signed-off-by: MichaelMorris <michael.morris@est.tech>
Issue-ID: SDC-3470
Change-Id: Iff835d230b173b9d44349caa6b0b11d783e8f8d3

4 files changed, 140 insertions, 5 deletions

diff --git a/catalog-model/pom.xml b/catalog-model/pom.xml
index 82d1eac6e8..27e40bf1f4 100644
--- a/catalog-model/pom.xml
+++ b/catalog-model/pom.xml
@@ -333,6 +333,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>com.vdurmont</groupId>
+      <artifactId>semver4j</artifactId>
+      <version>3.1.0</version>
+    </dependency>
 
   </dependencies>
   <build>
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ByToscaNameDerivedNodeTypeResolver.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ByToscaNameDerivedNodeTypeResolver.java
index a34dd7e920..9b9a3e6799 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ByToscaNameDerivedNodeTypeResolver.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ByToscaNameDerivedNodeTypeResolver.java
@@ -48,9 +48,8 @@ public class ByToscaNameDerivedNodeTypeResolver implements DerivedNodeTypeResolv
         propertiesToMatch.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
 
         propertiesToMatch.put(GraphPropertyEnum.TOSCA_RESOURCE_NAME, parentResource);
-        propertiesToMatch.put(GraphPropertyEnum.IS_HIGHEST_VERSION, true);
 
         return janusGraphDao
-            .getByCriteria(VertexTypeEnum.NODE_TYPE, propertiesToMatch, JsonParseFlagEnum.NoParse);
+            .getByCriteria(VertexTypeEnum.NODE_TYPE, propertiesToMatch, JsonParseFlagEnum.ParseMetadata);
     }
 }
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/NodeTypeOperation.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/NodeTypeOperation.java
index a67f92fff2..1860e696e5 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/NodeTypeOperation.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/NodeTypeOperation.java
@@ -26,6 +26,7 @@ import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -62,6 +63,8 @@ import org.openecomp.sdc.common.jsongraph.util.CommonUtility;
 import org.openecomp.sdc.common.jsongraph.util.CommonUtility.LogLevelEnum;
 import org.openecomp.sdc.common.log.wrappers.Logger;
 import org.springframework.beans.factory.annotation.Qualifier;
+import com.vdurmont.semver4j.Semver;
+import com.vdurmont.semver4j.Semver.SemverType;
 
 @org.springframework.stereotype.Component("node-type-operation")
 public class NodeTypeOperation extends ToscaElementOperation {
@@ -602,7 +605,7 @@ public class NodeTypeOperation extends ToscaElementOperation {
                         return Either.right(StorageOperationStatus.PARENT_RESOURCE_NOT_FOUND);
                     } else {
                         if (resources.size() > 1) {
-                            return handleMultipleParent(parentResource, derivedResources, resources);
+                            return handleMultipleParent(parentResource, derivedResources, resources, (String)nodeType.getMetadataValue(JsonPresentationFields.VENDOR_RELEASE));
                         } else {
                             GraphVertex parentResourceData = resources.get(0);
                             derivedResources.add(parentResourceData);
@@ -616,10 +619,11 @@ public class NodeTypeOperation extends ToscaElementOperation {
         return Either.left(derivedResources);
     }
 
-    Either<List<GraphVertex>, StorageOperationStatus> handleMultipleParent(String parentResource, List<GraphVertex> derivedResource, List<GraphVertex> fetchedDerivedResources) {
+    Either<List<GraphVertex>, StorageOperationStatus> handleMultipleParent(String parentResource, List<GraphVertex> derivedResource, List<GraphVertex> fetchedDerivedResources, String vendorRelease) {
 
         Either<List<GraphVertex>, StorageOperationStatus> result = Either.left(derivedResource);
         try {
+            fetchedDerivedResources.removeIf(graphVertex -> !isValidForVendorRelease(graphVertex, vendorRelease));
             fetchedDerivedResources.sort((d1, d2) -> {
                 return new Double(Double.parseDouble((String) d1.getMetadataProperty(GraphPropertyEnum.VERSION))).compareTo(Double.parseDouble((String) d2.getMetadataProperty(GraphPropertyEnum.VERSION)));
             });
@@ -638,6 +642,20 @@ public class NodeTypeOperation extends ToscaElementOperation {
         }
         return result;
     }
+    
+    private boolean isValidForVendorRelease(final GraphVertex resource, final String vendorRelease) {
+        if (vendorRelease != null && !vendorRelease.equals("1.0")) {
+            try {
+                Semver resourceSemVer = new Semver((String)resource.getJsonMetadataField(JsonPresentationFields.VENDOR_RELEASE), SemverType.NPM);
+                Semver packageSemVer = new Semver(vendorRelease, SemverType.NPM);
+                return !resourceSemVer.isGreaterThan(packageSemVer); 
+            } catch (Exception exception) {
+                log.debug("Error in comparing vendor release", exception);
+                return false;
+            }
+        }
+        return true;
+    }
 
     private StorageOperationStatus fixMultipleParent(List<GraphVertex> fetchedDerivedResources) {
         StorageOperationStatus result = StorageOperationStatus.OK;
diff --git a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ToscaOperationFacade.java b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ToscaOperationFacade.java
index b16e129d2e..cdddb2049f 100644
--- a/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ToscaOperationFacade.java
+++ b/catalog-model/src/main/java/org/openecomp/sdc/be/model/jsonjanusgraph/operations/ToscaOperationFacade.java
@@ -23,7 +23,9 @@ package org.openecomp.sdc.be.model.jsonjanusgraph.operations;
 import static java.util.Objects.requireNonNull;
 import static org.apache.commons.collections.CollectionUtils.isEmpty;
 import static org.apache.commons.collections.CollectionUtils.isNotEmpty;
-
+import static org.janusgraph.core.attribute.Contain.NOT_IN;
+import static org.janusgraph.core.attribute.Text.REGEX;
+import static org.openecomp.sdc.be.dao.janusgraph.JanusGraphUtils.buildNotInPredicate;
 import fj.data.Either;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -47,6 +49,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.janusgraph.graphdb.query.JanusGraphPredicate;
 import org.openecomp.sdc.be.config.Configuration;
 import org.openecomp.sdc.be.config.ConfigurationManager;
 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
@@ -55,6 +58,7 @@ import org.openecomp.sdc.be.dao.jsongraph.HealingJanusGraphDao;
 import org.openecomp.sdc.be.dao.jsongraph.types.EdgeLabelEnum;
 import org.openecomp.sdc.be.dao.jsongraph.types.JsonParseFlagEnum;
 import org.openecomp.sdc.be.dao.jsongraph.types.VertexTypeEnum;
+import org.openecomp.sdc.be.dao.neo4j.GraphPropertiesDictionary;
 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
 import org.openecomp.sdc.be.datatypes.elements.CapabilityDataDefinition;
 import org.openecomp.sdc.be.datatypes.elements.ComponentInstanceDataDefinition;
@@ -74,6 +78,7 @@ import org.openecomp.sdc.be.datatypes.elements.RequirementDataDefinition;
 import org.openecomp.sdc.be.datatypes.enums.ComponentTypeEnum;
 import org.openecomp.sdc.be.datatypes.enums.GraphPropertyEnum;
 import org.openecomp.sdc.be.datatypes.enums.JsonPresentationFields;
+import org.openecomp.sdc.be.datatypes.enums.NodeTypeEnum;
 import org.openecomp.sdc.be.datatypes.enums.OriginTypeEnum;
 import org.openecomp.sdc.be.datatypes.enums.PromoteVersionEnum;
 import org.openecomp.sdc.be.datatypes.enums.ResourceTypeEnum;
@@ -113,11 +118,14 @@ import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
 import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder;
 import org.openecomp.sdc.be.model.utils.GroupUtils;
 import org.openecomp.sdc.be.resources.data.ComponentMetadataData;
+import org.openecomp.sdc.be.resources.data.GroupTypeData;
 import org.openecomp.sdc.common.jsongraph.util.CommonUtility;
 import org.openecomp.sdc.common.jsongraph.util.CommonUtility.LogLevelEnum;
 import org.openecomp.sdc.common.log.wrappers.Logger;
 import org.openecomp.sdc.common.util.ValidationUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import com.vdurmont.semver4j.Semver;
+import com.vdurmont.semver4j.Semver.SemverType;
 
 
 @org.springframework.stereotype.Component("tosca-operation-facade")
@@ -418,6 +426,87 @@ public class ToscaOperationFacade {
 
         return getLatestCertifiedByToscaResourceName(toscaResourceName, VertexTypeEnum.NODE_TYPE, JsonParseFlagEnum.ParseMetadata);
     }
+        
+    public Either<Resource, StorageOperationStatus> getByToscaResourceNameMatchingVendorRelease(final String toscaResourceName, final String vendorVersion) {
+
+        return getByToscaResourceNameMatchingVendorRelease(toscaResourceName, VertexTypeEnum.NODE_TYPE, JsonParseFlagEnum.ParseMetadata, vendorVersion);
+    }
+    
+    public Either<Resource, StorageOperationStatus> getByToscaResourceNameMatchingVendorRelease(String toscaResourceName,
+        VertexTypeEnum vertexType, JsonParseFlagEnum parseFlag, String vendorRelease) {
+
+            Map<GraphPropertyEnum, Object> props = new EnumMap<>(GraphPropertyEnum.class);
+            props.put(GraphPropertyEnum.TOSCA_RESOURCE_NAME, toscaResourceName);
+            props.put(GraphPropertyEnum.STATE, LifecycleStateEnum.CERTIFIED.name());
+            
+            Map<String, Entry<JanusGraphPredicate,  Object>> predicateCriteria = getVendorVersionPredicate(vendorRelease);
+
+            Either<List<GraphVertex>, JanusGraphOperationStatus> getLatestRes = janusGraphDao
+                .getByCriteria(vertexType, props, null, predicateCriteria, parseFlag);
+            
+            if(getLatestRes.isRight() || CollectionUtils.isEmpty(getLatestRes.left().value())) {
+                getLatestRes = janusGraphDao.getByCriteria(vertexType, props, parseFlag);
+            }
+
+            return getLatestRes
+                .right().map(
+                    status -> {
+                        CommonUtility.addRecordToLog(log, LogLevelEnum.DEBUG, "Failed to fetch {} with name {}. status={} ",
+                            vertexType, toscaResourceName, status);
+                        return DaoStatusConverter.convertJanusGraphStatusToStorageStatus(status);
+                    }
+                )
+                .left().bind(
+                    resources -> {
+                        double version = 0.0;
+                        GraphVertex highestResource = null;
+                        for (GraphVertex resource : resources) {
+                            double resourceVersion = Double
+                                .parseDouble((String) resource.getJsonMetadataField(JsonPresentationFields.VERSION));
+                            if (resourceVersion > version && isValidForVendorRelease(resource, vendorRelease)) {
+                                version = resourceVersion;
+                                highestResource = resource;
+                            }
+                        }
+                        if (highestResource != null) {
+                            return getToscaFullElement(highestResource.getUniqueId());
+                        } else {
+                            log.debug("The vertex with the highest version could not be found for {}", toscaResourceName);
+                            return Either.right(StorageOperationStatus.GENERAL_ERROR);
+                        }
+                    }
+                );
+    }
+     
+     private Map<String, Entry<JanusGraphPredicate, Object>> getVendorVersionPredicate(final String vendorRelease) {
+         Map<String, Entry<JanusGraphPredicate,  Object>> predicateCriteria = new HashMap<>();
+         if (vendorRelease != null && vendorRelease != "1.0") {
+             String[] vendorReleaseElements = vendorRelease.split("\\.");
+             if (vendorReleaseElements.length > 0) {
+                 String regex = ".*\"vendorRelease\":\"";
+                 for (int i = 0; i <vendorReleaseElements.length; i++) {
+                     regex += vendorReleaseElements[i];
+                     regex += i < vendorReleaseElements.length -1 ? "\\." : "\".*";
+                 }
+                 predicateCriteria.put("metadata", new HashMap.SimpleEntry<>(REGEX, regex));
+             }
+         } 
+         return predicateCriteria;
+     }
+     
+     private boolean isValidForVendorRelease(final GraphVertex resource, final String vendorRelease) {
+         if (!vendorRelease.equals("1.0")) {
+             try {
+                 Semver resourceSemVer = new Semver((String)resource.getJsonMetadataField(JsonPresentationFields.VENDOR_RELEASE), SemverType.NPM);
+                 Semver packageSemVer = new Semver(vendorRelease, SemverType.NPM);
+                 return !resourceSemVer.isGreaterThan(packageSemVer); 
+             } catch (Exception exception) {
+                 log.debug("Error in comparing vendor release", exception);
+                 return true;
+             }
+         }
+         return true;
+     }
 
     public Either<Resource, StorageOperationStatus> getLatestCertifiedByToscaResourceName(String toscaResourceName,
         VertexTypeEnum vertexType, JsonParseFlagEnum parseFlag) {
@@ -785,6 +874,30 @@ public class ToscaOperationFacade {
         }
         return getToscaElementByOperation(getResourceRes.left().value().get(0));
     }
+    
+    public <T extends Component> Either<T, StorageOperationStatus> getComponentByNameAndVendorRelease(
+            final ComponentTypeEnum componentType, final String name, final String vendorRelease,
+            final JsonParseFlagEnum parseFlag) {
+
+        Map<GraphPropertyEnum, Object> hasProperties = new EnumMap<>(GraphPropertyEnum.class);
+        Map<GraphPropertyEnum, Object> hasNotProperties = new EnumMap<>(GraphPropertyEnum.class);
+
+        hasProperties.put(GraphPropertyEnum.NAME, name);
+        hasNotProperties.put(GraphPropertyEnum.IS_DELETED, true);
+        if (componentType != null) {
+            hasProperties.put(GraphPropertyEnum.COMPONENT_TYPE, componentType.name());
+        }
+        Map<String, Entry<JanusGraphPredicate,  Object>> predicateCriteria = getVendorVersionPredicate(vendorRelease);
+
+        Either<List<GraphVertex>, JanusGraphOperationStatus> getResourceRes = janusGraphDao
+            .getByCriteria(null, hasProperties, hasNotProperties, predicateCriteria, parseFlag);
+        if (getResourceRes.isRight()) {
+            JanusGraphOperationStatus status = getResourceRes.right().value();
+            log.debug("failed to find resource with name {}, version {}. Status is {} ", name, predicateCriteria, status);
+            return Either.right(DaoStatusConverter.convertJanusGraphStatusToStorageStatus(status));
+        }
+        return getToscaElementByOperation(getResourceRes.left().value().get(0));
+    }
 
     public Either<List<CatalogComponent>, StorageOperationStatus> getCatalogOrArchiveComponents(boolean isCatalog, List<OriginTypeEnum> excludeTypes) {
         List<ResourceTypeEnum> excludedResourceTypes = Optional.ofNullable(excludeTypes).orElse(Collections.emptyList()).stream().filter(type -> !type.equals(OriginTypeEnum.SERVICE)).map(type -> ResourceTypeEnum.getTypeByName(type.name()))