Include custom data types from VSP in csar imports

When a model with custom data types is created, the data types
are now being added as part of the default model imports files.
When a CSAR is generated from a Service Template that is
associated to a derived model, the default imports from the
parent model are also included in the package.

Change-Id: I027d25d7237989a40085edec7fdd399ac09c4db1
Issue-ID: SDC-3692
Signed-off-by: andre.schmid <andre.schmid@est.tech>

5 files changed, 82 insertions, 62 deletions

diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogic.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogic.java
index 3951be2d58..c80f993a71 100644
--- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogic.java
+++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogic.java
@@ -52,9 +52,26 @@ public class ModelBusinessLogic {
         this.dataTypeImportManager = dataTypeImportManager;
     }
 
+    /**
+     * Creates a model along with given data types. The data types must be provided in a yaml format, where each entry is one data type object, for
+     * example:
+     * <pre>
+     * tosca.datatypes.TimeInterval:
+     *   derived_from: tosca.datatypes.Root
+     *   [...]
+     *
+     * tosca.datatypes.network.NetworkInfo:
+     *   derived_from: tosca.datatypes.Root
+     *   [...]
+     * </pre>
+     *
+     * @param model         the model to create
+     * @param datatypesYaml the data types to create in yaml format. It can contain multiple data types entries.
+     * @return the created model.
+     */
     public Model createModel(final Model model, final String datatypesYaml) {
         createModel(model);
-        dataTypeImportManager.createDataTypes(datatypesYaml, model.getName(), false);
+        dataTypeImportManager.createDataTypes(datatypesYaml, model.getName(), true);
         return model;
     }
 
diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java
index 823ddc48ff..73906deb34 100644
--- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java
+++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/CsarUtils.java
@@ -62,10 +62,11 @@ import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.output.ByteArrayOutputStream;
-import org.apache.commons.text.WordUtils;
+import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.ImmutableTriple;
 import org.apache.commons.lang3.tuple.Triple;
+import org.apache.commons.text.WordUtils;
 import org.onap.sdc.tosca.services.YamlUtil;
 import org.openecomp.sdc.be.components.impl.ImportUtils;
 import org.openecomp.sdc.be.components.impl.exceptions.ByResponseFormatComponentException;
@@ -77,7 +78,6 @@ import org.openecomp.sdc.be.dao.api.ActionStatus;
 import org.openecomp.sdc.be.dao.cassandra.ArtifactCassandraDao;
 import org.openecomp.sdc.be.dao.cassandra.CassandraOperationStatus;
 import org.openecomp.sdc.be.dao.cassandra.SdcSchemaFilesCassandraDao;
-import org.openecomp.sdc.be.dao.cassandra.ToscaModelImportCassandraDao;
 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
 import org.openecomp.sdc.be.datatypes.elements.ArtifactDataDefinition;
 import org.openecomp.sdc.be.datatypes.elements.OperationDataDefinition;
@@ -95,6 +95,7 @@ import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade
 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
 import org.openecomp.sdc.be.model.operations.impl.DaoStatusConverter;
+import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
 import org.openecomp.sdc.be.plugins.CsarEntryGenerator;
 import org.openecomp.sdc.be.resources.data.DAOArtifactData;
 import org.openecomp.sdc.be.tosca.utils.OperationArtifactUtil;
@@ -103,6 +104,7 @@ import org.openecomp.sdc.common.api.ArtifactGroupTypeEnum;
 import org.openecomp.sdc.common.api.ArtifactTypeEnum;
 import org.openecomp.sdc.common.impl.ExternalConfiguration;
 import org.openecomp.sdc.common.log.elements.LoggerSupportability;
+import org.openecomp.sdc.common.log.enums.EcompLoggerErrorCode;
 import org.openecomp.sdc.common.log.enums.LoggerSupportabilityActions;
 import org.openecomp.sdc.common.log.enums.StatusCode;
 import org.openecomp.sdc.common.log.wrappers.Logger;
@@ -121,8 +123,6 @@ public class CsarUtils {
 
     public static final String NODES_YML = "nodes.yml";
     public static final String ARTIFACTS_PATH = "Artifacts/";
-    public static final String WORKFLOW_ARTIFACT_DIR = "Workflows" + File.separator + "BPMN" + File.separator;
-    public static final String DEPLOYMENT_ARTIFACTS_DIR = "Deployment" + File.separator;
     public static final String ARTIFACTS = "Artifacts";
     public static final String ARTIFACT_CREATED_FROM_CSAR = "Artifact created from csar";
     private static final Logger log = Logger.getLogger(CsarUtils.class);
@@ -160,33 +160,41 @@ public class CsarUtils {
             + VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS + DEL_PATTERN + VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS + DEL_PATTERN
             + VALID_ENGLISH_ARTIFACT_NAME_WITH_DIGITS;
     private static final String BLOCK_0_TEMPLATE = "SDC-TOSCA-Meta-File-Version: %s\nSDC-TOSCA-Definitions-Version: %s\n";
+
+    private final ToscaOperationFacade toscaOperationFacade;
+    private final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
+    private final ArtifactCassandraDao artifactCassandraDao;
+    private final ComponentsUtils componentsUtils;
+    private final ToscaExportHandler toscaExportUtils;
+    private final List<CsarEntryGenerator> generators;
+    private final ModelOperation modelOperation;
+    private final String versionFirstThreeOctets;
+
     @Autowired
-    protected ToscaOperationFacade toscaOperationFacade;
-    @Autowired
-    private SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao;
-    @Autowired
-    private ArtifactCassandraDao artifactCassandraDao;
-    @Autowired
-    private ComponentsUtils componentsUtils;
-    @Autowired
-    private ToscaExportHandler toscaExportUtils;
-    @Autowired(required = false)
-    private List<CsarEntryGenerator> generators;
-    @Autowired(required = false)
-    private ToscaModelImportCassandraDao toscaModelImportCassandraDao;
-    private String versionFirstThreeOctets;
-
-    public CsarUtils() {
-        if (SDC_VERSION != null && !SDC_VERSION.isEmpty()) {
-            // change regex to avoid DoS sonar issue
-            Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
-            matcher.find();
-            setVersionFirstThreeOctets(matcher.group(0));
-        } else {
-            setVersionFirstThreeOctets("");
+    public CsarUtils(final ToscaOperationFacade toscaOperationFacade, final SdcSchemaFilesCassandraDao sdcSchemaFilesCassandraDao,
+                     final ArtifactCassandraDao artifactCassandraDao, final ComponentsUtils componentsUtils,
+                     final ToscaExportHandler toscaExportUtils, final List<CsarEntryGenerator> generators, final ModelOperation modelOperation) {
+        this.toscaOperationFacade = toscaOperationFacade;
+        this.sdcSchemaFilesCassandraDao = sdcSchemaFilesCassandraDao;
+        this.artifactCassandraDao = artifactCassandraDao;
+        this.componentsUtils = componentsUtils;
+        this.toscaExportUtils = toscaExportUtils;
+        this.generators = generators;
+        this.modelOperation = modelOperation;
+        this.versionFirstThreeOctets = readVersionFirstThreeOctets();
+    }
+
+    private String readVersionFirstThreeOctets() {
+        if (StringUtils.isEmpty(SDC_VERSION)) {
+            return "";
         }
+        // change regex to avoid DoS sonar issue
+        Matcher matcher = Pattern.compile("(?!\\.)(\\d{1,9}(\\.\\d{1,9}){1,9})(?![\\d\\.])").matcher(SDC_VERSION);
+        matcher.find();
+        return matcher.group(0);
     }
 
+
     private static <L, R> F<L, Either<L, R>> iff(Predicate<L> p, Function<L, Either<L, R>> ifTrue) {
         return l -> p.test(l) ? ifTrue.apply(l) : Either.left(l);
     }
@@ -809,18 +817,19 @@ public class CsarUtils {
             .bind(iff(List::isEmpty, () -> schemaFileFetchError(fto), s -> Either.left(s.iterator().next().getPayloadAsArray())));
     }
 
-    private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String model) {
+    private void addSchemaFilesByModel(final ZipOutputStream zipOutputStream, final String modelName) {
         try {
-            final List<ToscaImportByModel> schemaImportsByModel = toscaModelImportCassandraDao.findAllByModel(model);
-            for (ToscaImportByModel toscaImportByModel : schemaImportsByModel) {
-                final ZipEntry zipEntry = new ZipEntry(DEFINITIONS_PATH + toscaImportByModel.getFullPath());
+            final List<ToscaImportByModel> modelDefaultImportList = modelOperation.findAllModelImports(modelName, true);
+            for (final ToscaImportByModel toscaImportByModel : modelDefaultImportList) {
+                final var zipEntry = new ZipEntry(DEFINITIONS_PATH + toscaImportByModel.getFullPath());
                 zipOutputStream.putNextEntry(zipEntry);
                 final byte[] content = toscaImportByModel.getContent().getBytes(StandardCharsets.UTF_8);
                 zipOutputStream.write(content, 0, content.length);
                 zipOutputStream.closeEntry();
             }
-        } catch (IOException e) {
-            log.error("Error while writing the schema files by model to the CSAR " + "{}", e);
+        } catch (final IOException e) {
+            log.error(EcompLoggerErrorCode.BUSINESS_PROCESS_ERROR, CsarUtils.class.getName(),
+                "Error while writing the schema files by model to the CSAR", e);
             throw new ByResponseFormatComponentException(componentsUtils.getResponseFormat(ActionStatus.CSAR_TOSCA_IMPORTS_ERROR));
         }
     }
@@ -1232,10 +1241,6 @@ public class CsarUtils {
         return versionFirstThreeOctets;
     }
 
-    public void setVersionFirstThreeOctets(String versionFirstThreeOctetes) {
-        this.versionFirstThreeOctets = versionFirstThreeOctetes;
-    }
-
     private Map<String, List<ArtifactDefinition>> getComponentInstanceSpecificArtifacts(Map<String, ArtifactDefinition> componentArtifacts,
                                                                                         Map<ArtifactGroupTypeEnum, Map<String, List<ArtifactDefinition>>> componentTypeArtifacts,
                                                                                         ArtifactGroupTypeEnum artifactGroupTypeEnum) {
diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java
index b70d676200..1b27857869 100644
--- a/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java
+++ b/catalog-be/src/main/java/org/openecomp/sdc/be/tosca/ToscaExportHandler.java
@@ -44,7 +44,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import lombok.NoArgsConstructor;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.FilenameUtils;
@@ -56,7 +55,6 @@ import org.onap.sdc.tosca.services.YamlUtil;
 import org.openecomp.sdc.be.components.impl.exceptions.SdcResourceNotFoundException;
 import org.openecomp.sdc.be.config.Configuration;
 import org.openecomp.sdc.be.config.ConfigurationManager;
-import org.openecomp.sdc.be.dao.cassandra.ToscaModelImportCassandraDao;
 import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus;
 import org.openecomp.sdc.be.data.model.ToscaImportByModel;
 import org.openecomp.sdc.be.datatypes.components.ResourceMetadataDataDefinition;
@@ -102,6 +100,7 @@ import org.openecomp.sdc.be.model.jsonjanusgraph.operations.ToscaOperationFacade
 import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter;
 import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus;
 import org.openecomp.sdc.be.model.operations.impl.InterfaceLifecycleOperation;
+import org.openecomp.sdc.be.model.operations.impl.ModelOperation;
 import org.openecomp.sdc.be.tosca.PropertyConvertor.PropertyType;
 import org.openecomp.sdc.be.tosca.builder.ToscaRelationshipBuilder;
 import org.openecomp.sdc.be.tosca.exception.ToscaConversionException;
@@ -142,7 +141,6 @@ import org.yaml.snakeyaml.nodes.Tag;
 import org.yaml.snakeyaml.representer.Represent;
 import org.yaml.snakeyaml.representer.Representer;
 
-@NoArgsConstructor
 @org.springframework.stereotype.Component("tosca-export-handler")
 public class ToscaExportHandler {
 
@@ -159,19 +157,18 @@ public class ToscaExportHandler {
     private static final List<String> EXCLUDED_CATEGORY_SPECIFIC_METADATA = List
         .of("Service Function", "Service Role", "Naming Policy", "Service Type");
     private static final YamlUtil yamlUtil = new YamlUtil();
-    private static final String COULD_NOT_PARSE_COMPONENT_ATTRIBUTES_COMPONENT_UNIQUE_ID = "Could not parse component '{}' attributes. Component unique id '{}'.";
-    private ApplicationDataTypeCache applicationDataTypeCache;
-    private ToscaOperationFacade toscaOperationFacade;
-    private CapabilityRequirementConverter capabilityRequirementConverter;
-    private PolicyExportParser policyExportParser;
-    private GroupExportParser groupExportParser;
-    private PropertyConvertor propertyConvertor;
-    private AttributeConverter attributeConverter;
-    private InputConverter inputConverter;
-    private OutputConverter outputConverter;
-    private InterfaceLifecycleOperation interfaceLifecycleOperation;
-    private InterfacesOperationsConverter interfacesOperationsConverter;
-    private ToscaModelImportCassandraDao toscaModelImportCassandraDao;
+    private final ApplicationDataTypeCache applicationDataTypeCache;
+    private final ToscaOperationFacade toscaOperationFacade;
+    private final CapabilityRequirementConverter capabilityRequirementConverter;
+    private final PolicyExportParser policyExportParser;
+    private final GroupExportParser groupExportParser;
+    private final PropertyConvertor propertyConvertor;
+    private final AttributeConverter attributeConverter;
+    private final InputConverter inputConverter;
+    private final OutputConverter outputConverter;
+    private final InterfaceLifecycleOperation interfaceLifecycleOperation;
+    private final InterfacesOperationsConverter interfacesOperationsConverter;
+    private final ModelOperation modelOperation;
 
     @Autowired
     public ToscaExportHandler(final ApplicationDataTypeCache applicationDataTypeCache,
@@ -185,7 +182,7 @@ public class ToscaExportHandler {
                               final OutputConverter outputConverter,
                               final InterfaceLifecycleOperation interfaceLifecycleOperation,
                               final InterfacesOperationsConverter interfacesOperationsConverter,
-                              final ToscaModelImportCassandraDao toscaModelImportCassandraDao) {
+                              final ModelOperation modelOperation) {
         this.applicationDataTypeCache = applicationDataTypeCache;
         this.toscaOperationFacade = toscaOperationFacade;
         this.capabilityRequirementConverter = capabilityRequirementConverter;
@@ -197,7 +194,7 @@ public class ToscaExportHandler {
         this.outputConverter = outputConverter;
         this.interfaceLifecycleOperation = interfaceLifecycleOperation;
         this.interfacesOperationsConverter = interfacesOperationsConverter;
-        this.toscaModelImportCassandraDao = toscaModelImportCassandraDao;
+        this.modelOperation = modelOperation;
     }
 
     public static String getInterfaceFilename(String artifactName) {
@@ -303,13 +300,14 @@ public class ToscaExportHandler {
         }
     }
 
-    public List<Map<String, Map<String, String>>> getDefaultToscaImports(final String model) {
-        if (model == null) {
+    public List<Map<String, Map<String, String>>> getDefaultToscaImports(final String modelId) {
+        if (modelId == null) {
             return getDefaultToscaImportConfig();
         }
-        final List<ToscaImportByModel> importsByModel = toscaModelImportCassandraDao.findAllByModel(model);
+        
+        final List<ToscaImportByModel> allModelImports = modelOperation.findAllModelImports(modelId, true);
         final List<Map<String, Map<String, String>>> importList = new ArrayList<>();
-        for(ToscaImportByModel toscaImportByModel: importsByModel) {
+        for(final ToscaImportByModel toscaImportByModel: allModelImports) {
             final String fileName = FilenameUtils.getBaseName(toscaImportByModel.getFullPath());
             importList.add(Map.of(fileName, Map.of("file", toscaImportByModel.getFullPath())));
         }
diff --git a/catalog-be/src/test/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogicTest.java b/catalog-be/src/test/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogicTest.java
index 5e9d4e7f5d..cbb0de8715 100644
--- a/catalog-be/src/test/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogicTest.java
+++ b/catalog-be/src/test/java/org/openecomp/sdc/be/components/impl/ModelBusinessLogicTest.java
@@ -89,7 +89,7 @@ class ModelBusinessLogicTest {
         assertThat(result).isNotNull();
         assertThat(result.getName()).isEqualTo(model.getName());
 
-        verify(dataTypeImportManager).createDataTypes(dataTypes, model.getName(), false);
+        verify(dataTypeImportManager).createDataTypes(dataTypes, model.getName(), true);
     }
 
     @Test
diff --git a/catalog-be/src/test/java/org/openecomp/sdc/be/tosca/InterfacesOperationsConverterTest.java b/catalog-be/src/test/java/org/openecomp/sdc/be/tosca/InterfacesOperationsConverterTest.java
index 074d70bfe0..d20f1c0600 100644
--- a/catalog-be/src/test/java/org/openecomp/sdc/be/tosca/InterfacesOperationsConverterTest.java
+++ b/catalog-be/src/test/java/org/openecomp/sdc/be/tosca/InterfacesOperationsConverterTest.java
@@ -237,7 +237,7 @@ class InterfacesOperationsConverterTest {
             .getInterfacesMap(component, null, component.getInterfaces(), null, false, true);
         ToscaNodeType nodeType = new ToscaNodeType();
         nodeType.setInterfaces(interfacesMap);
-        ToscaExportHandler handler = new ToscaExportHandler();
+        ToscaExportHandler handler = new ToscaExportHandler(null, null, null, null, null, null, null, null, null, null, null, null);
         ToscaTemplate template = new ToscaTemplate(NODE_TYPE_NAME);
         Map<String, ToscaNodeType> nodeTypes = new HashMap<>();
         nodeTypes.put(NODE_TYPE_NAME, nodeType);