From 0f6ae59460165381a8b5da2eea58c2c2efa75f2f Mon Sep 17 00:00:00 2001 From: MichaelMorris Date: Mon, 25 Jul 2022 23:01:50 +0100 Subject: Support unknown data types in service import Signed-off-by: MichaelMorris Issue-ID: SDC-4119 Change-Id: I426ebb96e7b354dfd5dfb06a10845c3162055968 --- .../sdc/be/components/csar/CsarBusinessLogic.java | 8 +- .../openecomp/sdc/be/components/csar/CsarInfo.java | 171 +++-------------- .../sdc/be/components/csar/OnboardedCsarInfo.java | 205 +++++++++++++++++++++ .../sdc/be/components/csar/ServiceCsarInfo.java | 145 +++++++++++++++ .../be/components/impl/DataTypeImportManager.java | 64 ++++++- .../be/components/impl/ResourceBusinessLogic.java | 5 +- .../impl/ServiceImportBusinessLogic.java | 41 ++++- 7 files changed, 477 insertions(+), 162 deletions(-) create mode 100644 catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/OnboardedCsarInfo.java create mode 100644 catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/ServiceCsarInfo.java (limited to 'catalog-be/src/main') diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarBusinessLogic.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarBusinessLogic.java index 1d37efef28..aa5706c77f 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarBusinessLogic.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarBusinessLogic.java @@ -106,7 +106,7 @@ public class CsarBusinessLogic extends BaseBusinessLogic { } } - public CsarInfo getCsarInfo(Resource resource, Resource oldResource, User user, Map payload, String csarUUID) { + public OnboardedCsarInfo getCsarInfo(Resource resource, Resource oldResource, User user, Map payload, String csarUUID) { Map csar = payload; if (csar == null) { final var vendorSoftwareProduct = getCsar(resource, user); @@ -125,7 +125,7 @@ public class CsarBusinessLogic extends BaseBusinessLogic { csarUUID, oldResource.getComponentMetadataDefinition().getMetadataDataDefinition().getImportedToscaChecksum(), checksum); oldResource.getComponentMetadataDefinition().getMetadataDataDefinition().setImportedToscaChecksum(checksum); } - return new CsarInfo(user, csarUUID, resource.getCsarVersionId(), csar, resource.getName(), toscaYamlCsarStatus.getKey(), + return new OnboardedCsarInfo(user, csarUUID, resource.getCsarVersionId(), csar, resource.getName(), toscaYamlCsarStatus.getKey(), toscaYamlCsarStatus.getValue(), true); } @@ -144,7 +144,7 @@ public class CsarBusinessLogic extends BaseBusinessLogic { } } - public CsarInfo getCsarInfo(Service service, Service oldResource, User user, Map payload, String csarUUID) { + public ServiceCsarInfo getCsarInfo(Service service, Service oldResource, User user, Map payload, String csarUUID) { Map csar = getCsar(service, user, payload, csarUUID); ImmutablePair toscaYamlCsarStatus = validateAndParseCsar(service, user, csar, csarUUID).left() .on(this::throwComponentException); @@ -157,7 +157,7 @@ public class CsarBusinessLogic extends BaseBusinessLogic { csarUUID, oldResource.getComponentMetadataDefinition().getMetadataDataDefinition().getImportedToscaChecksum(), checksum); oldResource.getComponentMetadataDefinition().getMetadataDataDefinition().setImportedToscaChecksum(checksum); } - return new CsarInfo(user, csarUUID, csar, service.getName(), toscaYamlCsarStatus.getKey(), toscaYamlCsarStatus.getValue(), true); + return new ServiceCsarInfo(user, csarUUID, csar, service.getName(), toscaYamlCsarStatus.getKey(), toscaYamlCsarStatus.getValue(), true); } public ParsedToscaYamlInfo getParsedToscaYamlInfo(String topologyTemplateYaml, String yamlName, Map nodeTypesInfo, diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarInfo.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarInfo.java index dc6f7c4416..49b7bb2f86 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarInfo.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/CsarInfo.java @@ -21,8 +21,6 @@ */ package org.openecomp.sdc.be.components.csar; -import static org.openecomp.sdc.be.components.impl.ImportUtils.ResultStatusEnum; -import static org.openecomp.sdc.be.components.impl.ImportUtils.ToscaElementTypeEnum; import static org.openecomp.sdc.be.components.impl.ImportUtils.findToscaElement; import com.google.common.annotations.VisibleForTesting; @@ -37,11 +35,11 @@ import java.util.Optional; import java.util.PriorityQueue; import java.util.Queue; import java.util.Set; -import java.util.regex.Pattern; import lombok.Getter; import lombok.Setter; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.openecomp.sdc.be.components.impl.ImportUtils.ResultStatusEnum; +import org.openecomp.sdc.be.components.impl.ImportUtils.ToscaElementTypeEnum; import org.openecomp.sdc.be.components.impl.exceptions.ByActionStatusComponentException; import org.openecomp.sdc.be.config.NonManoArtifactType; import org.openecomp.sdc.be.config.NonManoConfiguration; @@ -51,13 +49,15 @@ import org.openecomp.sdc.be.dao.api.ActionStatus; import org.openecomp.sdc.be.model.NodeTypeInfo; import org.openecomp.sdc.be.model.Resource; import org.openecomp.sdc.be.model.User; -import org.openecomp.sdc.be.tosca.CsarUtils; import org.openecomp.sdc.be.utils.TypeUtils; import org.openecomp.sdc.common.api.Constants; import org.openecomp.sdc.common.log.wrappers.Logger; import org.yaml.snakeyaml.Yaml; -public class CsarInfo { +/** + * Provides access to the contents of a CSAR + */ +public abstract class CsarInfo { private static final Logger log = Logger.getLogger(CsarInfo.class); private final NonManoConfiguration nonManoConfiguration; @@ -74,22 +74,21 @@ public class CsarInfo { private String csarVersionId; @Getter @Setter - private Map csar; + protected Map csar; @Getter private String mainTemplateName; @Getter private String mainTemplateContent; @Getter - private Map mappedToscaMainTemplate; + protected Map mappedToscaMainTemplate; @Getter private Map createdNodesToscaResourceNames; private Queue cvfcToCreateQueue; private boolean isUpdate; @Getter private Map createdNodes; - private Map datatypeDefinitions; + protected Map datatypeDefinitions; private Map policytypeDefinitions; - private List> globalSubstitutes; public CsarInfo(User modifier, String csarUUID, Map csar, String vfResourceName, String mainTemplateName, @@ -106,18 +105,11 @@ public class CsarInfo { this.isUpdate = isUpdate; this.createdNodes = new HashMap<>(); this.nonManoConfiguration = NonManoConfigurationManager.getInstance().getNonManoConfiguration(); - this.globalSubstitutes = getGlobalSubstitutes(csar); } - private List> getGlobalSubstitutes(final Map csar){ - final List> globalSubstitutesInCsar = new ArrayList<>(); - for (Map.Entry entry : csar.entrySet()) { - if (isAServiceTemplate(entry.getKey()) && isGlobalSubstitute(entry.getKey())) { - globalSubstitutesInCsar.add(entry); - } - } - return globalSubstitutesInCsar; - } + public String getVfResourceName() { + return vfResourceName; + } public CsarInfo(final User modifier, final String csarUUID, final String csarVersionId, final Map csarContent, final String vfResourceName, final String mainTemplateName, final String mainTemplateContent, final boolean isUpdate) { @@ -174,34 +166,15 @@ public class CsarInfo { public void setUpdate(boolean isUpdate) { this.isUpdate = isUpdate; } - - public Map extractTypesInfo() { - Map nodeTypesInfo = new HashMap<>(); - final Set nodeTypesUsedInNodeTemplates = new HashSet<>(); - for (Map.Entry entry : getCsar().entrySet()) { - extractNodeTypeInfo(nodeTypesInfo, nodeTypesUsedInNodeTemplates, entry); - } - if (CollectionUtils.isNotEmpty(globalSubstitutes)) { - setDerivedFrom(nodeTypesInfo); - addGlobalSubstitutionsToNodeTypes(nodeTypesUsedInNodeTemplates, nodeTypesInfo); - } - - markNestedVfc(getMappedToscaMainTemplate(), nodeTypesInfo); - return nodeTypesInfo; - } - public Map getDataTypes() { - if (datatypeDefinitions == null) { - datatypeDefinitions = new HashMap<>(); - for (Map.Entry entry : globalSubstitutes) { - final String yamlFileContents = new String(entry.getValue()); - final Map mappedToscaTemplate = new Yaml().load(yamlFileContents); - datatypeDefinitions.putAll(getTypesFromTemplate(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.DATA_TYPES)); - } - datatypeDefinitions.putAll(getTypesFromTemplate(mappedToscaMainTemplate, TypeUtils.ToscaTagNamesEnum.DATA_TYPES)); - } - return datatypeDefinitions; - } + public abstract Map extractTypesInfo(); + + /** + * Get the data types defined in the CSAR + * + * @return map with the data type name as key and representaion of the data type defintion as value + */ + public abstract Map getDataTypes(); public Map getPolicyTypes() { if (policytypeDefinitions == null) { @@ -211,8 +184,8 @@ public class CsarInfo { return policytypeDefinitions; } - @SuppressWarnings("unchecked") - private Map getTypesFromTemplate(final Map mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum type) { + @SuppressWarnings("unchecked") + protected Map getTypesFromTemplate(final Map mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum type) { final Either dataTypesEither = findToscaElement(mappedToscaTemplate, type, ToscaElementTypeEnum.MAP); if (dataTypesEither != null && dataTypesEither.isLeft()) { @@ -221,27 +194,7 @@ public class CsarInfo { return Collections.emptyMap(); } - @SuppressWarnings("unchecked") - private void extractNodeTypeInfo(Map nodeTypesInfo, - final Set nodeTypesUsedInNodeTemplates, Map.Entry entry) { - if (isAServiceTemplate(entry.getKey()) && !isGlobalSubstitute(entry.getKey())) { - Map mappedToscaTemplate = (Map) new Yaml().load(new String(entry.getValue())); - findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.SUBSTITUTION_MAPPINGS, ToscaElementTypeEnum.MAP).right() - .on(sub -> handleSubstitutionMappings(nodeTypesInfo, entry, mappedToscaTemplate, (Map) sub)); - final Either nodeTypesEither = findToscaElement(mappedToscaTemplate, - TypeUtils.ToscaTagNamesEnum.NODE_TEMPLATES, ToscaElementTypeEnum.MAP); - if (nodeTypesEither.isLeft()) { - final Map> nodeTemplates = (Map>) nodeTypesEither.left().value(); - nodeTypesUsedInNodeTemplates.addAll(findNodeTypesUsedInNodeTemplates(nodeTemplates)); - } - } - } - - private boolean isAServiceTemplate(final String filePath) { - return Pattern.compile(CsarUtils.SERVICE_TEMPLATE_PATH_PATTERN).matcher(filePath).matches(); - } - - private Set findNodeTypesUsedInNodeTemplates(final Map> nodeTemplates) { + protected Set findNodeTypesUsedInNodeTemplates(final Map> nodeTemplates) { final Set nodeTypes = new HashSet<>(); for (final Map nodeTemplate : nodeTemplates.values()) { nodeTypes.add((String) nodeTemplate.get(TypeUtils.ToscaTagNamesEnum.TYPE.getElementName())); @@ -249,84 +202,8 @@ public class CsarInfo { return nodeTypes; } - private ResultStatusEnum handleSubstitutionMappings(Map nodeTypesInfo, Map.Entry entry, - Map mappedToscaTemplate, Map substitutionMappings) { - final Set nodeTypesDefinedInTemplate = findNodeTypesDefinedInTemplate(mappedToscaTemplate); - if (substitutionMappings.containsKey(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName()) && !nodeTypesDefinedInTemplate - .contains(substitutionMappings.get(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName()))) { - NodeTypeInfo nodeTypeInfo = new NodeTypeInfo(); - nodeTypeInfo.setSubstitutionMapping(true); - nodeTypeInfo.setType((String) substitutionMappings.get(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName())); - nodeTypeInfo.setTemplateFileName(entry.getKey()); - nodeTypeInfo.setMappedToscaTemplate(mappedToscaTemplate); - nodeTypesInfo.put(nodeTypeInfo.getType(), nodeTypeInfo); - } - return ResultStatusEnum.OK; - } - - @SuppressWarnings("unchecked") - private Set findNodeTypesDefinedInTemplate(final Map mappedToscaTemplate) { - final Either nodeTypesEither = findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, - ToscaElementTypeEnum.MAP); - if (nodeTypesEither.isLeft()) { - final Map nodeTypes = (Map) nodeTypesEither.left().value(); - return nodeTypes.keySet(); - } - return Collections.emptySet(); - } - - private boolean isGlobalSubstitute(String fileName) { - return fileName.equalsIgnoreCase(Constants.GLOBAL_SUBSTITUTION_TYPES_SERVICE_TEMPLATE) || fileName - .equalsIgnoreCase(Constants.ABSTRACT_SUBSTITUTE_GLOBAL_TYPES_SERVICE_TEMPLATE); - } - - @SuppressWarnings("unchecked") - private void setDerivedFrom(Map nodeTypesInfo) { - for (Map.Entry entry : globalSubstitutes) { - String yamlFileContents = new String(entry.getValue()); - Map mappedToscaTemplate = (Map) new Yaml().load(yamlFileContents); - Either nodeTypesEither = findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, - ToscaElementTypeEnum.MAP); - if (nodeTypesEither.isLeft()) { - Map nodeTypes = (Map) nodeTypesEither.left().value(); - for (Map.Entry nodeType : nodeTypes.entrySet()) { - processNodeType(nodeTypesInfo, nodeType); - } - } - } - } - - @SuppressWarnings("unchecked") - private void processNodeType(Map nodeTypesInfo, Map.Entry nodeType) { - Map nodeTypeMap = (Map) nodeType.getValue(); - if (nodeTypeMap.containsKey(TypeUtils.ToscaTagNamesEnum.DERIVED_FROM.getElementName()) && nodeTypesInfo.containsKey(nodeType.getKey())) { - NodeTypeInfo nodeTypeInfo = nodeTypesInfo.get(nodeType.getKey()); - List derivedFrom = new ArrayList<>(); - derivedFrom.add((String) nodeTypeMap.get(TypeUtils.ToscaTagNamesEnum.DERIVED_FROM.getElementName())); - nodeTypeInfo.setDerivedFrom(derivedFrom); - } - } - - @SuppressWarnings("unchecked") - private void addGlobalSubstitutionsToNodeTypes(final Set nodeTypesUsedInNodeTemplates, final Map nodeTypesInfo) { - for (Map.Entry entry : globalSubstitutes) { - final String yamlFileContents = new String(entry.getValue()); - final Map mappedToscaTemplate = (Map) new Yaml().load(yamlFileContents); - final Either nodeTypesEither = findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, - ToscaElementTypeEnum.MAP); - if (nodeTypesEither.isLeft()) { - final Map nodeTypes = (Map) nodeTypesEither.left().value(); - for (final Map.Entry nodeType : nodeTypes.entrySet()) { - if (!nodeTypesInfo.containsKey(nodeType.getKey()) && nodeTypesUsedInNodeTemplates.contains(nodeType.getKey())) { - nodeTypesInfo.put(nodeType.getKey(), buildNodeTypeInfo(nodeType, entry.getKey(), mappedToscaTemplate)); - } - } - } - } - } - @SuppressWarnings("unchecked") - private NodeTypeInfo buildNodeTypeInfo(final Map.Entry nodeType, final String templateFileName, + protected NodeTypeInfo buildNodeTypeInfo(final Map.Entry nodeType, final String templateFileName, final Map mappedToscaTemplate) { final NodeTypeInfo nodeTypeInfo = new NodeTypeInfo(); nodeTypeInfo.setSubstitutionMapping(false); diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/OnboardedCsarInfo.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/OnboardedCsarInfo.java new file mode 100644 index 0000000000..85f64699d4 --- /dev/null +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/OnboardedCsarInfo.java @@ -0,0 +1,205 @@ +/* + * - + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.components.csar; + +import static org.openecomp.sdc.be.components.impl.ImportUtils.findToscaElement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import org.apache.commons.collections.CollectionUtils; +import org.openecomp.sdc.be.components.impl.ImportUtils.ResultStatusEnum; +import org.openecomp.sdc.be.components.impl.ImportUtils.ToscaElementTypeEnum; +import org.openecomp.sdc.be.config.NonManoConfiguration; +import org.openecomp.sdc.be.model.NodeTypeInfo; +import org.openecomp.sdc.be.model.User; +import org.openecomp.sdc.be.tosca.CsarUtils; +import org.openecomp.sdc.be.utils.TypeUtils; +import org.openecomp.sdc.common.api.Constants; +import org.yaml.snakeyaml.Yaml; +import fj.data.Either; + +/** + * Provides access to the contents of a CSAR which has been created through the SDC onboarding + * process + */ +public class OnboardedCsarInfo extends CsarInfo { + + private List> globalSubstitutes; + + OnboardedCsarInfo(NonManoConfiguration nonManoConfiguration) { + super(nonManoConfiguration); + } + + public OnboardedCsarInfo(final User modifier, final String csarUUID, final Map csar, final String vfResourceName, + final String mainTemplateName, final String mainTemplateContent, final boolean isUpdate) { + super(modifier, csarUUID, csar, vfResourceName, mainTemplateName, mainTemplateContent, isUpdate); + this.globalSubstitutes = getGlobalSubstitutes(csar); + } + + public OnboardedCsarInfo(final User modifier, final String csarUUID, final String csarVersionId, final Map csarContent, + final String vfResourceName, final String mainTemplateName, final String mainTemplateContent, final boolean isUpdate) { + super(modifier, csarUUID, csarVersionId, csarContent, vfResourceName, mainTemplateName, mainTemplateContent, isUpdate); + this.globalSubstitutes = getGlobalSubstitutes(csar); + } + + private List> getGlobalSubstitutes(final Map csar) { + final List> globalSubstitutesInCsar = new ArrayList<>(); + for (Map.Entry entry : csar.entrySet()) { + if (isAServiceTemplate(entry.getKey()) && isGlobalSubstitute(entry.getKey())) { + globalSubstitutesInCsar.add(entry); + } + } + return globalSubstitutesInCsar; + } + + public Map extractTypesInfo() { + final Map nodeTypesInfo = new HashMap<>(); + final Set nodeTypesUsedInNodeTemplates = new HashSet<>(); + for (Map.Entry entry : getCsar().entrySet()) { + extractNodeTypeInfo(nodeTypesInfo, nodeTypesUsedInNodeTemplates, entry); + } + if (CollectionUtils.isNotEmpty(globalSubstitutes)) { + setDerivedFrom(nodeTypesInfo); + addGlobalSubstitutionsToNodeTypes(nodeTypesUsedInNodeTemplates, nodeTypesInfo); + } + + markNestedVfc(getMappedToscaMainTemplate(), nodeTypesInfo); + return nodeTypesInfo; + } + + @SuppressWarnings("unchecked") + private void extractNodeTypeInfo(final Map nodeTypesInfo, final Set nodeTypesUsedInNodeTemplates, + final Map.Entry entry) { + if (isAServiceTemplate(entry.getKey()) && !isGlobalSubstitute(entry.getKey())) { + final Map mappedToscaTemplate = (Map) new Yaml().load(new String(entry.getValue())); + findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.SUBSTITUTION_MAPPINGS, ToscaElementTypeEnum.MAP).right() + .on(sub -> handleSubstitutionMappings(nodeTypesInfo, entry, mappedToscaTemplate, (Map) sub)); + final Either nodeTypesEither = + findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TEMPLATES, ToscaElementTypeEnum.MAP); + if (nodeTypesEither.isLeft()) { + final Map> nodeTemplates = (Map>) nodeTypesEither.left().value(); + nodeTypesUsedInNodeTemplates.addAll(findNodeTypesUsedInNodeTemplates(nodeTemplates)); + } + } + } + + private boolean isAServiceTemplate(final String filePath) { + return Pattern.compile(CsarUtils.SERVICE_TEMPLATE_PATH_PATTERN).matcher(filePath).matches(); + } + + private boolean isGlobalSubstitute(final String fileName) { + return fileName.equalsIgnoreCase(Constants.GLOBAL_SUBSTITUTION_TYPES_SERVICE_TEMPLATE) + || fileName.equalsIgnoreCase(Constants.ABSTRACT_SUBSTITUTE_GLOBAL_TYPES_SERVICE_TEMPLATE); + } + + + private ResultStatusEnum handleSubstitutionMappings(final Map nodeTypesInfo, final Map.Entry entry, + final Map mappedToscaTemplate, final Map substitutionMappings) { + final Set nodeTypesDefinedInTemplate = findNodeTypesDefinedInTemplate(mappedToscaTemplate); + if (substitutionMappings.containsKey(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName()) + && !nodeTypesDefinedInTemplate.contains(substitutionMappings.get(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName()))) { + NodeTypeInfo nodeTypeInfo = new NodeTypeInfo(); + nodeTypeInfo.setSubstitutionMapping(true); + nodeTypeInfo.setType((String) substitutionMappings.get(TypeUtils.ToscaTagNamesEnum.NODE_TYPE.getElementName())); + nodeTypeInfo.setTemplateFileName(entry.getKey()); + nodeTypeInfo.setMappedToscaTemplate(mappedToscaTemplate); + nodeTypesInfo.put(nodeTypeInfo.getType(), nodeTypeInfo); + } + return ResultStatusEnum.OK; + } + + @SuppressWarnings("unchecked") + private Set findNodeTypesDefinedInTemplate(final Map mappedToscaTemplate) { + final Either nodeTypesEither = + findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, ToscaElementTypeEnum.MAP); + if (nodeTypesEither.isLeft()) { + final Map nodeTypes = (Map) nodeTypesEither.left().value(); + return nodeTypes.keySet(); + } + return Collections.emptySet(); + } + + @SuppressWarnings("unchecked") + private void setDerivedFrom(final Map nodeTypesInfo) { + for (Map.Entry entry : globalSubstitutes) { + final String yamlFileContents = new String(entry.getValue()); + final Map mappedToscaTemplate = (Map) new Yaml().load(yamlFileContents); + Either nodeTypesEither = + findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, ToscaElementTypeEnum.MAP); + if (nodeTypesEither.isLeft()) { + Map nodeTypes = (Map) nodeTypesEither.left().value(); + for (Map.Entry nodeType : nodeTypes.entrySet()) { + processNodeType(nodeTypesInfo, nodeType); + } + } + } + } + + @SuppressWarnings("unchecked") + private void processNodeType(final Map nodeTypesInfo, final Map.Entry nodeType) { + final Map nodeTypeMap = (Map) nodeType.getValue(); + if (nodeTypeMap.containsKey(TypeUtils.ToscaTagNamesEnum.DERIVED_FROM.getElementName()) && nodeTypesInfo.containsKey(nodeType.getKey())) { + final NodeTypeInfo nodeTypeInfo = nodeTypesInfo.get(nodeType.getKey()); + final List derivedFrom = new ArrayList<>(); + derivedFrom.add((String) nodeTypeMap.get(TypeUtils.ToscaTagNamesEnum.DERIVED_FROM.getElementName())); + nodeTypeInfo.setDerivedFrom(derivedFrom); + } + } + + @SuppressWarnings("unchecked") + private void addGlobalSubstitutionsToNodeTypes(final Set nodeTypesUsedInNodeTemplates, final Map nodeTypesInfo) { + for (Map.Entry entry : globalSubstitutes) { + final String yamlFileContents = new String(entry.getValue()); + final Map mappedToscaTemplate = (Map) new Yaml().load(yamlFileContents); + final Either nodeTypesEither = + findToscaElement(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.NODE_TYPES, ToscaElementTypeEnum.MAP); + if (nodeTypesEither.isLeft()) { + final Map nodeTypes = (Map) nodeTypesEither.left().value(); + for (final Map.Entry nodeType : nodeTypes.entrySet()) { + if (!nodeTypesInfo.containsKey(nodeType.getKey()) && nodeTypesUsedInNodeTemplates.contains(nodeType.getKey())) { + nodeTypesInfo.put(nodeType.getKey(), buildNodeTypeInfo(nodeType, entry.getKey(), mappedToscaTemplate)); + } + } + } + } + } + + @Override + public Map getDataTypes() { + if (datatypeDefinitions == null) { + datatypeDefinitions = new HashMap<>(); + for (Map.Entry entry : globalSubstitutes) { + final String yamlFileContents = new String(entry.getValue()); + final Map mappedToscaTemplate = new Yaml().load(yamlFileContents); + datatypeDefinitions.putAll(getTypesFromTemplate(mappedToscaTemplate, TypeUtils.ToscaTagNamesEnum.DATA_TYPES)); + } + datatypeDefinitions.putAll(getTypesFromTemplate(mappedToscaMainTemplate, TypeUtils.ToscaTagNamesEnum.DATA_TYPES)); + } + return datatypeDefinitions; + } + +} diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/ServiceCsarInfo.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/ServiceCsarInfo.java new file mode 100644 index 0000000000..ca3c92bcbd --- /dev/null +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/csar/ServiceCsarInfo.java @@ -0,0 +1,145 @@ +/* + * - + * ============LICENSE_START======================================================= + * Copyright (C) 2022 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.openecomp.sdc.be.components.csar; + +import static org.openecomp.sdc.be.components.impl.ImportUtils.findToscaElement; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.openecomp.sdc.be.components.impl.ImportUtils.ResultStatusEnum; +import org.openecomp.sdc.be.components.impl.ImportUtils.ToscaElementTypeEnum; +import org.openecomp.sdc.be.model.NodeTypeInfo; +import org.openecomp.sdc.be.model.User; +import org.openecomp.sdc.be.utils.TypeUtils; +import org.openecomp.sdc.be.utils.TypeUtils.ToscaTagNamesEnum; +import org.openecomp.sdc.common.log.wrappers.Logger; +import org.yaml.snakeyaml.Yaml; +import fj.data.Either; + +/** + * Provides access to the contents of a Service CSAR + */ +public class ServiceCsarInfo extends CsarInfo { + + private Map> mainTemplateImports; + private static final Logger log = Logger.getLogger(ServiceCsarInfo.class); + + public ServiceCsarInfo(final User modifier, final String csarUUID, final Map csar, final String vfResourceName, + final String mainTemplateName, final String mainTemplateContent, final boolean isUpdate) { + super(modifier, csarUUID, csar, vfResourceName, mainTemplateName, mainTemplateContent, isUpdate); + + final Path mainTemplateDir = Paths.get(getMainTemplateName().substring(0, getMainTemplateName().lastIndexOf('/') + 1)); + final Collection filesHandled = new HashSet<>(); + filesHandled.add(Paths.get(mainTemplateName)); + this.mainTemplateImports = getTemplateImports(csar, new Yaml().load(mainTemplateContent), mainTemplateDir, filesHandled); + } + + private Map> getTemplateImports(final Map csar, Map mappedToscaMainTemplate, + final Path fileParentDir, final Collection filesHandled) { + final Map> templateImports = new HashMap<>(); + + final List importFilePaths = getTempateImportFilePaths(mappedToscaMainTemplate, fileParentDir); + + importFilePaths.stream().filter(path -> !filesHandled.contains(path)).forEach( + importFilePath -> { + byte[] importFile = csar.get(importFilePath.toString()); + if (importFile != null) { + filesHandled.add(importFilePath); + Map mappedImportFile = new Yaml().load(new String(csar.get(importFilePath.toString()))); + templateImports.put(importFilePath.toString(), mappedImportFile); + + templateImports.putAll(getTemplateImports(csar, mappedImportFile, importFilePath.getParent(), filesHandled)); + + } else { + log.info("Import {} cannot be found in CSAR", importFilePath.toString()); + } + }); + + return templateImports; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private List getTempateImportFilePaths(final Map mappedToscaTemplate, final Path fileParentDir) { + final Either importsEither = + findToscaElement(mappedToscaTemplate, ToscaTagNamesEnum.IMPORTS, ToscaElementTypeEnum.ALL); + + if (importsEither.isLeft()) { + final List importsList = (List) importsEither.left().value(); + if (CollectionUtils.isNotEmpty(importsList)) { + if (importsList.get(0) instanceof String) { + List importPaths = new ArrayList<>(); + importsList.stream().forEach(importPath -> importPaths.add(Paths.get((String)importPath))); + return importPaths; + } else if (importsList.get(0) instanceof Map) { + return getTemplateImportFilePathsMultiLineGrammar(importsList, fileParentDir); + } + } + + } + return Collections.emptyList(); + } + + @SuppressWarnings("unchecked") + private List getTemplateImportFilePathsMultiLineGrammar(final List> importsList, final Path fileParentDir) { + final List importFiles = new ArrayList<>(); + + for (Map importFileMultiLineGrammar : (List>) importsList) { + if (MapUtils.isNotEmpty(importFileMultiLineGrammar)) { + if (importFileMultiLineGrammar.values().iterator().next() instanceof String) { + Path relativePath = Paths.get((String) importFileMultiLineGrammar.get("file")); + Path absolutePath = fileParentDir.resolve(relativePath).normalize(); + importFiles.add(absolutePath); + } else if (importFileMultiLineGrammar.values().iterator().next() instanceof Map) { + importFileMultiLineGrammar.values().forEach(value -> { + Path relativePath = Paths.get((String) ((Map) value).get("file")); + Path absolutePath = fileParentDir == null ? relativePath : fileParentDir.resolve(relativePath).normalize(); + importFiles.add(absolutePath); + }); + } + } + } + return importFiles; + } + + @Override + public Map extractTypesInfo() { + return Collections.emptyMap(); + } + + @Override + public Map getDataTypes() { + final Map definitions = new HashMap<>(); + mainTemplateImports.entrySet().stream() + .forEach(entry -> definitions.putAll(getTypesFromTemplate(entry.getValue(), TypeUtils.ToscaTagNamesEnum.DATA_TYPES))); + definitions.putAll(getTypesFromTemplate(getMappedToscaMainTemplate(), TypeUtils.ToscaTagNamesEnum.DATA_TYPES)); + return definitions; + } + +} diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/DataTypeImportManager.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/DataTypeImportManager.java index cad0f28477..60fdc0b3d5 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/DataTypeImportManager.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/DataTypeImportManager.java @@ -17,10 +17,13 @@ * limitations under the License. * ============LICENSE_END========================================================= */ + package org.openecomp.sdc.be.components.impl; import fj.data.Either; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -72,17 +75,66 @@ public class DataTypeImportManager { } private Either, ActionStatus> createDataTypesFromYml(final String dataTypesYml, final String modelName) { - final Either, ActionStatus> dataTypes = commonImportManager.createElementTypesFromYml(dataTypesYml, + final Either, ActionStatus> dataTypesEither = commonImportManager.createElementTypesFromYml(dataTypesYml, this::createDataType); - if (dataTypes.isLeft() && StringUtils.isNotEmpty(modelName)) { + if (dataTypesEither.isRight()) { + return dataTypesEither; + } + final List dataTypes = dataTypesEither.left().value(); + if (StringUtils.isNotEmpty(modelName)) { final Optional modelOptional = modelOperation.findModelByName(modelName); if (modelOptional.isPresent()) { - dataTypes.left().value().forEach(dataType -> dataType.setModel(modelName)); - return dataTypes; + dataTypes.forEach(dataType -> dataType.setModel(modelName)); + } else { + return Either.right(ActionStatus.INVALID_MODEL); + } + } + if (log.isTraceEnabled()) { + log.trace("Unsorted datatypes order:"); + dataTypes.stream().forEach(dt -> log.trace(dt.getName())); + } + + long startTime = System.currentTimeMillis(); + List sortedDataTypeDefinitions = sortDataTypesByDependencyOrder(dataTypes); + + if (log.isTraceEnabled()) { + long sortTime = System.currentTimeMillis() - startTime; + log.trace("Sorting " + sortedDataTypeDefinitions.size() + " data types from model: " + modelName + " took: " + sortTime); + log.trace("Sorted datatypes order:"); + sortedDataTypeDefinitions.stream().forEach(dt -> log.trace(dt.getName())); + } + return Either.left(sortedDataTypeDefinitions); + } + + private List sortDataTypesByDependencyOrder(final List dataTypes) { + final List sortedDataTypeDefinitions = new ArrayList<>(); + final Map dataTypeDefinitionsMap = new HashMap<>(); + + dataTypes.forEach(dataType -> { + + int highestDependencyIndex = -1; + for (final String dependencyName : getDependencyTypes(dataType)) { + final DataTypeDefinition dependency = dataTypeDefinitionsMap.get(dependencyName); + final int indexOfDependency = sortedDataTypeDefinitions.lastIndexOf(dependency); + highestDependencyIndex = indexOfDependency > highestDependencyIndex ? indexOfDependency : highestDependencyIndex; } - return Either.right(ActionStatus.INVALID_MODEL); + sortedDataTypeDefinitions.add(highestDependencyIndex + 1, dataType); + dataTypeDefinitionsMap.put(dataType.getName(), dataType); + + } ); + + return sortedDataTypeDefinitions; + } + + private Collection getDependencyTypes(final DataTypeDefinition dataType) { + final Set dependencies = new HashSet<>(); + if (dataType.getDerivedFromName() != null) { + dependencies.add(dataType.getDerivedFromName()); + } + if (dataType.getProperties() != null) { + dataType.getProperties().stream().forEach(property -> dependencies.add(property.getType())); } - return dataTypes; + return dependencies; } private Either>, ResponseFormat> createDataTypesByDao( diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ResourceBusinessLogic.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ResourceBusinessLogic.java index 1f107e4ec1..b923cbc3a1 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ResourceBusinessLogic.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ResourceBusinessLogic.java @@ -58,6 +58,7 @@ import org.openecomp.sdc.be.catalog.enums.ChangeTypeEnum; import org.openecomp.sdc.be.components.csar.CsarArtifactsAndGroupsBusinessLogic; import org.openecomp.sdc.be.components.csar.CsarBusinessLogic; import org.openecomp.sdc.be.components.csar.CsarInfo; +import org.openecomp.sdc.be.components.csar.OnboardedCsarInfo; import org.openecomp.sdc.be.components.impl.ArtifactsBusinessLogic.ArtifactOperationEnum; import org.openecomp.sdc.be.components.impl.ImportUtils.ResultStatusEnum; import org.openecomp.sdc.be.components.impl.artifact.ArtifactOperationInfo; @@ -1041,7 +1042,8 @@ public class ResourceBusinessLogic extends ComponentBusinessLogic { loggerSupportability .log(LoggerSupportabilityActions.CREATE_RESOURCE_FROM_YAML, StatusCode.STARTED, "Starting to create Resource From Csar by user {}", user.getUserId()); - CsarInfo csarInfo = csarBusinessLogic.getCsarInfo(resource, null, user, csarUIPayload, csarUUID); + OnboardedCsarInfo csarInfo = csarBusinessLogic.getCsarInfo(resource, null, user, csarUIPayload, csarUUID); + Map nodeTypesInfo = csarInfo.extractTypesInfo(); final String model = resource.getModel(); if (StringUtils.isNotEmpty(model)) { @@ -1057,7 +1059,6 @@ public class ResourceBusinessLogic extends ComponentBusinessLogic { policyTypeBusinessLogic.createPolicyTypeFromYaml(new Yaml().dump(policyTypesToCreate), model, true); } } - Either>>, ResponseFormat> findNodeTypesArtifactsToHandleRes = findNodeTypesArtifactsToHandle( nodeTypesInfo, csarInfo, resource); if (findNodeTypesArtifactsToHandleRes.isRight()) { diff --git a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ServiceImportBusinessLogic.java b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ServiceImportBusinessLogic.java index 7e15a52b90..e4129a24eb 100644 --- a/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ServiceImportBusinessLogic.java +++ b/catalog-be/src/main/java/org/openecomp/sdc/be/components/impl/ServiceImportBusinessLogic.java @@ -48,6 +48,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.openecomp.sdc.be.components.csar.CsarArtifactsAndGroupsBusinessLogic; import org.openecomp.sdc.be.components.csar.CsarBusinessLogic; import org.openecomp.sdc.be.components.csar.CsarInfo; +import org.openecomp.sdc.be.components.csar.ServiceCsarInfo; import org.openecomp.sdc.be.components.impl.ArtifactsBusinessLogic.ArtifactOperationEnum; import org.openecomp.sdc.be.components.impl.artifact.ArtifactOperationInfo; import org.openecomp.sdc.be.components.impl.exceptions.BusinessLogicException; @@ -62,6 +63,7 @@ import org.openecomp.sdc.be.config.BeEcompErrorManager; import org.openecomp.sdc.be.config.ConfigurationManager; import org.openecomp.sdc.be.dao.api.ActionStatus; import org.openecomp.sdc.be.dao.janusgraph.JanusGraphDao; +import org.openecomp.sdc.be.dao.janusgraph.JanusGraphOperationStatus; import org.openecomp.sdc.be.datamodel.utils.ArtifactUtils; import org.openecomp.sdc.be.datamodel.utils.UiComponentDataConverter; import org.openecomp.sdc.be.datatypes.elements.GetInputValueDataDefinition; @@ -118,6 +120,8 @@ import org.openecomp.sdc.be.model.jsonjanusgraph.utils.ModelConverter; import org.openecomp.sdc.be.model.operations.StorageException; import org.openecomp.sdc.be.model.operations.api.IGraphLockOperation; import org.openecomp.sdc.be.model.operations.api.StorageOperationStatus; +import org.openecomp.sdc.be.model.operations.impl.PropertyOperation; +import org.openecomp.sdc.be.model.operations.impl.UniqueIdBuilder; import org.openecomp.sdc.be.resources.data.auditing.AuditingActionEnum; import org.openecomp.sdc.be.tosca.CsarUtils; import org.openecomp.sdc.be.ui.model.OperationUi; @@ -130,6 +134,7 @@ import org.openecomp.sdc.common.kpi.api.ASDCKpiApi; import org.openecomp.sdc.common.log.wrappers.Logger; import org.openecomp.sdc.common.util.ValidationUtils; import org.openecomp.sdc.exception.ResponseFormat; +import org.springframework.beans.factory.annotation.Autowired; import org.yaml.snakeyaml.Yaml; @Getter @@ -167,6 +172,9 @@ public class ServiceImportBusinessLogic { private final ArtifactsBusinessLogic artifactsBusinessLogic; private final IGraphLockOperation graphLockOperation; private final ToscaFunctionService toscaFunctionService; + private final PropertyOperation propertyOperation; + private final DataTypeBusinessLogic dataTypeBusinessLogic; + private ApplicationDataTypeCache applicationDataTypeCache; public ServiceImportBusinessLogic(final GroupBusinessLogic groupBusinessLogic, final ArtifactsBusinessLogic artifactsBusinessLogic, @@ -180,7 +188,8 @@ public class ServiceImportBusinessLogic { final ServiceImportParseLogic serviceImportParseLogic, final ComponentNodeFilterBusinessLogic componentNodeFilterBusinessLogic, final PolicyBusinessLogic policyBusinessLogic, final JanusGraphDao janusGraphDao, - final IGraphLockOperation graphLockOperation, final ToscaFunctionService toscaFunctionService) { + final IGraphLockOperation graphLockOperation, final ToscaFunctionService toscaFunctionService, + final PropertyOperation propertyOperation, final DataTypeBusinessLogic dataTypeBusinessLogic) { this.componentInstanceBusinessLogic = componentInstanceBusinessLogic; this.uiComponentDataConverter = uiComponentDataConverter; this.componentsUtils = componentsUtils; @@ -199,6 +208,13 @@ public class ServiceImportBusinessLogic { this.artifactsBusinessLogic = artifactsBusinessLogic; this.graphLockOperation = graphLockOperation; this.toscaFunctionService = toscaFunctionService; + this.propertyOperation = propertyOperation; + this.dataTypeBusinessLogic = dataTypeBusinessLogic; + } + + @Autowired + public void setApplicationDataTypeCache(ApplicationDataTypeCache applicationDataTypeCache) { + this.applicationDataTypeCache = applicationDataTypeCache; } public Service createService(Service service, AuditingActionEnum auditingAction, User user, Map csarUIPayload, @@ -232,7 +248,13 @@ public class ServiceImportBusinessLogic { protected Service createServiceFromCsar(Service service, User user, Map csarUIPayload, String csarUUID) { log.trace("************* created successfully from YAML, resource TOSCA "); try { - CsarInfo csarInfo = csarBusinessLogic.getCsarInfo(service, null, user, csarUIPayload, csarUUID); + ServiceCsarInfo csarInfo = csarBusinessLogic.getCsarInfo(service, null, user, csarUIPayload, csarUUID); + + final Map dataTypesToCreate = getDatatypesToCreate(service.getModel(), csarInfo); + if (MapUtils.isNotEmpty(dataTypesToCreate)) { + dataTypeBusinessLogic.createDataTypeFromYaml(new Yaml().dump(dataTypesToCreate), service.getModel(), true); + } + Map nodeTypesInfo = csarInfo.extractTypesInfo(); Either>>, ResponseFormat> findNodeTypesArtifactsToHandleRes = serviceImportParseLogic .findNodeTypesArtifactsToHandle(nodeTypesInfo, csarInfo, service); @@ -250,6 +272,19 @@ public class ServiceImportBusinessLogic { throw new ComponentException(ActionStatus.GENERAL_ERROR); } } + + private Map getDatatypesToCreate(final String model, final CsarInfo csarInfo) { + final Map dataTypesToCreate = new HashMap<>(); + + for (final Entry dataTypeEntry : csarInfo.getDataTypes().entrySet()){ + final Either result = applicationDataTypeCache.get(model, UniqueIdBuilder.buildDataTypeUid(model, dataTypeEntry.getKey())); + if (result.isRight() && result.right().value().equals(JanusGraphOperationStatus.NOT_FOUND)) { + dataTypesToCreate.put(dataTypeEntry.getKey(), dataTypeEntry.getValue()); + log.info("Deploying unknown type " + dataTypeEntry.getKey() + " to model " + model + " from package " + csarInfo.getCsarUUID()); + } + } + return dataTypesToCreate; + } protected Service createServiceFromYaml(Service service, String topologyTemplateYaml, String yamlName, Map nodeTypesInfo, CsarInfo csarInfo, @@ -2545,7 +2580,7 @@ public class ServiceImportBusinessLogic { mapToConvert.put(TypeUtils.ToscaTagNamesEnum.TOSCA_VERSION.getElementName(), toscaVersion.left().value()); Map nodeTypes = serviceImportParseLogic.getNodeTypesFromTemplate(mappedToscaTemplate); createNodeTypes(yamlName, service, needLock, nodeTypesArtifactsToHandle, nodeTypesNewCreatedArtifacts, nodeTypesInfo, csarInfo, - mapToConvert, nodeTypes); + mapToConvert, nodeTypes); return csarInfo.getCreatedNodes(); } catch (Exception e) { log.debug("Exception occured when createResourcesFromYamlNodeTypesList,error is:{}", e.getMessage(), e); -- cgit 1.2.3-korg