diff options
8 files changed, 121 insertions, 232 deletions
diff --git a/catalog-be/src/main/resources/scripts/import/tosca/importHeatTypes1.py b/catalog-be/src/main/resources/scripts/import/tosca/importHeatTypes1.py deleted file mode 100644 index b89a94cbb6..0000000000 --- a/catalog-be/src/main/resources/scripts/import/tosca/importHeatTypes1.py +++ /dev/null @@ -1,97 +0,0 @@ -import pycurl -import sys, getopt -from StringIO import StringIO -import json -import copy -from importCommon import * -from importNormativeTypes import * -import importCommon - -################################################################################################################################################ -# # -# Import all users from a given file # -# # -# activation : # -# python importUsers.py [-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-f <input file> | --ifile=<input file> ] # -# # -# shortest activation (be host = localhost, be port = 8080): # -# python importUsers.py [-f <input file> | --ifile=<input file> ] # -# # -################################################################################################################################################ - -def importHeatTypes1(beHost, bePort, adminUser, fileDir, updateversion): - - heatTypes = ["extCp"] - - responseCodes = [200, 201] - - if(updateversion == 'false'): - responseCodes = [200, 201, 409] - - results = [] - for heatType in heatTypes: - result = createNormativeType(beHost, bePort, adminUser, fileDir, heatType, updateversion) - results.append(result) - if ( result[1] == None or result[1] not in responseCodes) : - print "Failed creating heat type " + heatType + ". " + str(result[1]) - return results - - -def main(argv): - print 'Number of arguments:', len(sys.argv), 'arguments.' - - beHost = 'localhost' - bePort = '8080' - adminUser = 'jh0003' - updateversion = 'true' - - try: - opts, args = getopt.getopt(argv,"i:p:u:v:h:",["ip=","port=","user=","updateversion="]) - except getopt.GetoptError: - usage() - errorAndExit(2, 'Invalid input') - - for opt, arg in opts: - #print opt, arg - if opt == '-h': - usage() - sys.exit(3) - elif opt in ("-i", "--ip"): - beHost = arg - elif opt in ("-p", "--port"): - bePort = arg - elif opt in ("-u", "--user"): - adminUser = arg - elif opt in ("-v", "--updateversion"): - if (arg.lower() == "false" or arg.lower() == "no"): - updateversion = 'false' - - print 'be host =',beHost,', be port =', bePort,', user =', adminUser - - if ( beHost == None ): - usage() - sys.exit(3) - - results = importHeatTypes(beHost, bePort, adminUser, "../../../import/tosca/heat-types/", updateversion) - - print "-----------------------------" - for result in results: - print "{0:20} | {1:6}".format(result[0], result[1]) - print "-----------------------------" - - responseCodes = [200, 201] - - if(updateversion == 'false'): - responseCodes = [200, 201, 409] - - failedNormatives = filter(lambda x: x[1] == None or x[1] not in responseCodes, results) - if (len(failedNormatives) > 0): - errorAndExit(1, None) - else: - errorAndExit(0, None) - - -if __name__ == "__main__": - main(sys.argv[1:]) - - diff --git a/catalog-be/src/main/resources/scripts/import/tosca/importNormativeAll1.py b/catalog-be/src/main/resources/scripts/import/tosca/importNormativeAll1.py deleted file mode 100644 index f564fc5b96..0000000000 --- a/catalog-be/src/main/resources/scripts/import/tosca/importNormativeAll1.py +++ /dev/null @@ -1,118 +0,0 @@ -import pycurl -import sys, getopt, os -from StringIO import StringIO -import json -import copy -import time -#from importNormativeElements import createNormativeElement -from importNormativeElements import * -from importNormativeTypes import importNormativeTypes -from importHeatTypes1 import importHeatTypes1 -from importNormativeCapabilities import importNormativeCapabilities -from importCategoryTypes import importCategories -from importNormativeInterfaceLifecycleTypes import importNormativeInterfaceLifecycleType -from importDataTypes import importDataTypes -from importGroupTypes import importGroupTypes -from importPolicyTypes import importPolicyTypes -from importCommon import * -import importCommon - -################################################################################################################################################################################################# -# # -# Import all users from a given file # -# # -# activation : # -# python importNormativeAll.py [-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-u <user userId> | --user=<user userId> ] [-d <true|false> | --debug=<true|false>] # -# [-v <true|false> | --updateversion=<true|false>] # -# # -# shortest activation (be host = localhost, be port = 8080, user = jh0003): # # # -# python importNormativeAll.py # -# # -################################################################################################################################################################################################# - -def usage(): - print sys.argv[0], '[-i <be host> | --ip=<be host>] [-p <be port> | --port=<be port> ] [-u <user userId> | --user=<user userId> ] [-d <true|false> | --debug=<true|false>] [-v <true|false> | --updateversion=<true|false>]' - -def handleResults(results, updateversion): - printFrameLine() - for result in results: - printNameAndReturnCode(result[0], result[1]) - printFrameLine() - - responseCodes = [200, 201] - - if(updateversion == 'false'): - responseCodes = [200, 201, 409] - - failedResults = filter(lambda x: x[1] == None or x[1] not in responseCodes, results) - if (len(failedResults) > 0): - errorAndExit(1, None) - -def main(argv): - print 'Number of arguments:', len(sys.argv), 'arguments.' - - beHost = 'localhost' - bePort = '8080' - adminUser = 'jh0003' - debugf = None - updateversion = 'true' - importCommon.debugFlag = False - - try: - opts, args = getopt.getopt(argv,"i:p:u:d:v:h",["ip=","port=","user=","debug=","updateversion="]) - except getopt.GetoptError: - usage() - errorAndExit(2, 'Invalid input') - - for opt, arg in opts: - #print opt, arg - if opt == '-h': - usage() - sys.exit(3) - elif opt in ("-i", "--ip"): - beHost = arg - elif opt in ("-p", "--port"): - bePort = arg - elif opt in ("-u", "--user"): - adminUser = arg - elif opt in ("-d", "--debug"): - print arg - debugf = bool(arg.lower() == "true" or arg.lower() == "yes") - elif opt in ("-v", "--updateversion"): - print arg - if (arg.lower() == "false" or arg.lower() == "no"): - updateversion = 'false' - - print 'be host =',beHost,', be port =', bePort,', user =', adminUser, ', debug =', debugf, ', updateversion =', updateversion - - if (debugf != None): - print 'set debug mode to ' + str(debugf) - importCommon.debugFlag = debugf - - if ( beHost == None ): - usage() - sys.exit(3) - - print sys.argv[0] - pathdir = os.path.dirname(os.path.realpath(sys.argv[0])) - debug("path dir =" + pathdir) - - baseFileLocation = pathdir + "/../../../import/tosca/" - - fileLocation = baseFileLocation + "data-types/" - importDataTypes(beHost, bePort, adminUser, False, fileLocation) - - print 'sleep until data type cache is updated' - time.sleep( 1 ) - - fileLocation = baseFileLocation + "heat-types/" - resultsHeat = importHeatTypes1(beHost, bePort, adminUser, fileLocation, updateversion) - handleResults(resultsHeat, updateversion) - - - - errorAndExit(0, None) - -if __name__ == "__main__": - main(sys.argv[1:]) - diff --git a/openecomp-be/lib/openecomp-heat-lib/src/main/java/org/openecomp/sdc/heat/services/tree/HeatTreeManager.java b/openecomp-be/lib/openecomp-heat-lib/src/main/java/org/openecomp/sdc/heat/services/tree/HeatTreeManager.java index 372c09cffe..df46ef048d 100644 --- a/openecomp-be/lib/openecomp-heat-lib/src/main/java/org/openecomp/sdc/heat/services/tree/HeatTreeManager.java +++ b/openecomp-be/lib/openecomp-heat-lib/src/main/java/org/openecomp/sdc/heat/services/tree/HeatTreeManager.java @@ -119,22 +119,19 @@ public class HeatTreeManager { } - private void handleHeatContentReference(String filename, HeatStructureTree fileHeatStructureTree, + private void handleHeatContentReference(HeatStructureTree fileHeatStructureTree, GlobalValidationContext globalContext) { String fileName = fileHeatStructureTree.getFileName(); - InputStream fileContent = this.heatContentMap.getFileContent(fileName); - if (fileContent == null) { - return; // file exist in manifest but does not exist in zip - } - try { + + try (InputStream fileContent = this.heatContentMap.getFileContent(fileName)) { HeatOrchestrationTemplate hot = new YamlUtil().yamlToObject(fileContent, HeatOrchestrationTemplate.class); - Set<String> nestedSet = HeatTreeManagerUtil.getNestedFiles(filename, hot, globalContext); + Set<String> nestedSet = HeatTreeManagerUtil.getNestedFiles(fileName, hot, globalContext); addHeatNestedFiles(fileHeatStructureTree, nestedSet); - Set<String> artifactSet = HeatTreeManagerUtil.getArtifactFiles(filename, hot, globalContext); + Set<String> artifactSet = HeatTreeManagerUtil.getArtifactFiles(fileName, hot, globalContext); addHeatArtifactFiles(fileHeatStructureTree, artifactSet); } catch (Exception ignore) { /* invalid yaml no need to process reference */ logger.debug("",ignore); @@ -224,7 +221,7 @@ public class HeatTreeManager { fileHeatStructureTree.setFileName(fileName); fileHeatStructureTree.setBase(fileData.getBase()); fileHeatStructureTree.setType(type); - handleHeatContentReference(null, fileHeatStructureTree, null); + handleHeatContentReference(fileHeatStructureTree, null); parentHeatStructureTree.addHeatToHeatList(fileHeatStructureTree); if (fileData.getData() != null) { scanTree(fileName, fileData.getData()); @@ -244,6 +241,7 @@ public class HeatTreeManager { if (fileData.getData() != null) { scanTree(fileName, fileData.getData()); } + handleHeatContentReference(childHeatStructureTree, null); } else if (FileData.Type.HEAT_VOL.equals(type)) { //parentHeatStructureTree.addVolumeFileToVolumeList(childHeatStructureTree); @@ -251,6 +249,7 @@ public class HeatTreeManager { if (fileData.getData() != null) { scanTree(fileName, fileData.getData()); } + handleHeatContentReference(childHeatStructureTree, null); } else if (FileData.Type.HEAT_ENV.equals(type)) { if (parentHeatStructureTree != null && parentHeatStructureTree.getFileName() != null) { parentHeatStructureTree.setEnv(childHeatStructureTree); diff --git a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/HeatToToscaUtil.java b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/HeatToToscaUtil.java index 81ef3e01ce..c1f69cf3bf 100644 --- a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/HeatToToscaUtil.java +++ b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/HeatToToscaUtil.java @@ -86,6 +86,7 @@ import org.openecomp.sdc.translator.services.heattotosca.globaltypes.GlobalTypes import org.openecomp.sdc.translator.services.heattotosca.helper.FunctionTranslationHelper; import org.openecomp.sdc.translator.services.heattotosca.mapping.TranslatorHeatToToscaPropertyConverter; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; @@ -135,11 +136,15 @@ public class HeatToToscaUtil { return translatorOutput; } - InputStream structureFile = getHeatStructureTreeFile(fileNameContentMap); - heatToToscaTranslator.addExternalArtifacts(SdcCommon.HEAT_META, structureFile); + try (InputStream structureFile = getHeatStructureTreeFile(fileNameContentMap)) { + heatToToscaTranslator.addExternalArtifacts(SdcCommon.HEAT_META, structureFile); - mdcDataDebugMessage.debugExitMessage(null, null); - return heatToToscaTranslator.translate(); + mdcDataDebugMessage.debugExitMessage(null, null); + return heatToToscaTranslator.translate(); + } catch (IOException e) { + // rethrow as a RuntimeException to keep the signature backward compatible + throw new RuntimeException(e); + } } diff --git a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/TranslationService.java b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/TranslationService.java index 43079f9025..012be05257 100644 --- a/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/TranslationService.java +++ b/openecomp-be/lib/openecomp-sdc-translator-lib/openecomp-sdc-translator-core/src/main/java/org/openecomp/sdc/translator/services/heattotosca/TranslationService.java @@ -20,6 +20,7 @@ package org.openecomp.sdc.translator.services.heattotosca; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.openecomp.core.translator.datatypes.TranslatorOutput; import org.openecomp.core.utilities.file.FileUtils; @@ -48,6 +49,7 @@ import org.openecomp.sdc.tosca.datatypes.model.TopologyTemplate; import org.openecomp.sdc.tosca.services.DataModelUtil; import org.openecomp.sdc.tosca.services.ToscaConstants; import org.openecomp.sdc.tosca.services.ToscaFileOutputService; +import org.openecomp.sdc.tosca.services.ToscaUtil; import org.openecomp.sdc.tosca.services.YamlUtil; import org.openecomp.sdc.tosca.services.impl.ToscaFileOutputServiceCsarImpl; import org.openecomp.sdc.translator.datatypes.heattotosca.AttachedResourceId; @@ -213,17 +215,54 @@ public class TranslationService { groupDefinition.getProperties().put(Constants.DESCRIPTION_PROPERTY_NAME, hotDescription); } groupDefinition.setMembers(new ArrayList<>()); - Map<String, Set<String>> heatStackGroupMembers = context.getHeatStackGroupMembers(); - if (heatStackGroupMembers.get(fileName) == null) { + Set<String> heatStackGroupMembersIds = getHeatStackGroupMembers(fileName, + serviceTemplate, context); + if (CollectionUtils.isEmpty(heatStackGroupMembersIds)) { return; //not creating a group when no resources are present in the heat input } - groupDefinition.getMembers().addAll(heatStackGroupMembers.get(fileName)); + groupDefinition.getMembers().addAll(heatStackGroupMembersIds); DataModelUtil .addGroupDefinitionToTopologyTemplate(serviceTemplate, heatStackGroupId, groupDefinition); mdcDataDebugMessage.debugExitMessage(null, null); } + private Set<String> getHeatStackGroupMembers(String heatFileName, + ServiceTemplate serviceTemplate, + TranslationContext context){ + + Map<String, Set<String>> heatStackGroupMembers = context.getHeatStackGroupMembers(); + Set<String> groupMembers = MapUtils.isEmpty(heatStackGroupMembers) ? new HashSet<>() + : heatStackGroupMembers.get(heatFileName); + + if(CollectionUtils.isEmpty(groupMembers)){ + return new HashSet<>(); + } + + Set<String> updatedMembersIds = new HashSet<>(); + + groupMembers.forEach(member -> { + if (Objects.nonNull(DataModelUtil.getNodeTemplate(serviceTemplate, member))) { + updatedMembersIds.add(member); + } else { + updateSubstitutableGroupMemberId(heatFileName, serviceTemplate, updatedMembersIds); + } + }); + + return updatedMembersIds; + } + + private void updateSubstitutableGroupMemberId(String heatFileName, + ServiceTemplate serviceTemplate, + Set<String> updatedMembersIds) { + Optional<String> substitutableGroupMemberId = + ToscaUtil.getSubstitutableGroupMemberId(heatFileName, serviceTemplate); + + if (substitutableGroupMemberId.isPresent()) { + updatedMembersIds.add(substitutableGroupMemberId.get()); + } + } + private void translateInputParameters(ServiceTemplate serviceTemplate, HeatOrchestrationTemplate heatOrchestrationTemplate, FileData heatFileData, TranslationContext context, diff --git a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/services/impl/filedatastructuremodule/CandidateServiceImpl.java b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/services/impl/filedatastructuremodule/CandidateServiceImpl.java index c18a0cbedd..4c38bd94a7 100644 --- a/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/services/impl/filedatastructuremodule/CandidateServiceImpl.java +++ b/openecomp-be/lib/openecomp-sdc-vendor-software-product-lib/openecomp-sdc-vendor-software-product-core/src/main/java/org/openecomp/sdc/vendorsoftwareproduct/services/impl/filedatastructuremodule/CandidateServiceImpl.java @@ -603,6 +603,7 @@ public class CandidateServiceImpl implements CandidateService { handleArtifactsFromTree(volume, structure); module.setVol(volume.getFileName()); handleEnv(module, volume, true, structure); + addNestedToFileDataStructure(volume, structure); } mdcDataDebugMessage.debugExitMessage(null); diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/DataModelUtil.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/DataModelUtil.java index 26596b994f..7c1dfdcafd 100644 --- a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/DataModelUtil.java +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/DataModelUtil.java @@ -198,6 +198,16 @@ public class DataModelUtil { mdcDataDebugMessage.debugExitMessage(null, null); } + public static Map<String, NodeTemplate> getNodeTemplates(ServiceTemplate serviceTemplate){ + if (Objects.isNull(serviceTemplate) + || Objects.isNull(serviceTemplate.getTopology_template()) + || MapUtils.isEmpty(serviceTemplate.getTopology_template().getNode_templates())){ + return new HashMap<>(); + } + + return serviceTemplate.getTopology_template().getNode_templates(); + } + /** * Add node template. * diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaUtil.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaUtil.java index f3bf3280dc..3b7db49fee 100644 --- a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaUtil.java +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaUtil.java @@ -20,10 +20,16 @@ package org.openecomp.sdc.tosca.services; +import org.apache.commons.collections4.MapUtils; +import org.openecomp.core.utilities.file.FileUtils; +import org.openecomp.sdc.tosca.datatypes.model.NodeTemplate; import org.openecomp.sdc.tosca.datatypes.model.ServiceTemplate; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.UUID; /** @@ -50,7 +56,7 @@ public class ToscaUtil { /** * Gets service template file name. * - * @param metaData the file name + * @param metadata the file name * @return the service template file name */ public static String getServiceTemplateFileName(Map<String, String> metadata) { @@ -63,6 +69,50 @@ public class ToscaUtil { } + public static Optional<String> getSubstitutableGroupMemberId(String heatFileName, + ServiceTemplate serviceTemplate){ + + Map<String, NodeTemplate> nodeTemplates = + DataModelUtil.getNodeTemplates(serviceTemplate); + + if(MapUtils.isEmpty(nodeTemplates)){ + return Optional.empty(); + } + + String heatFileNameWithoutExt = FileUtils.getFileWithoutExtention(heatFileName); + + for(Map.Entry<String, NodeTemplate> nodeTemplateEntry : nodeTemplates.entrySet()){ + String subServiceTemplateName = + getSubstitutionServiceTemplateNameFromProperties(nodeTemplateEntry); + + if(Objects.nonNull(subServiceTemplateName) + && isGroupMemberIdSubstitutable(heatFileNameWithoutExt, subServiceTemplateName)){ + return Optional.of(nodeTemplateEntry.getKey()); + } + } + + return Optional.empty(); + } + + private static boolean isGroupMemberIdSubstitutable(String heatFileNameWithoutExt, + String subServiceTemplateName) { + return subServiceTemplateName.startsWith(heatFileNameWithoutExt); + } + + private static String getSubstitutionServiceTemplateNameFromProperties( + Map.Entry<String, NodeTemplate> nodeTemplateEntry) { + Map<String, Object> properties = + nodeTemplateEntry.getValue().getProperties() == null ? Collections.emptyMap() : + nodeTemplateEntry.getValue().getProperties(); + + Map<String, Object> serviceTemplateFilter = + properties.containsKey(ToscaConstants.SERVICE_TEMPLATE_FILTER_PROPERTY_NAME)? + (Map<String, Object>) properties.get(ToscaConstants + .SERVICE_TEMPLATE_FILTER_PROPERTY_NAME) : Collections.emptyMap(); + + return (String) serviceTemplateFilter.get(ToscaConstants.SUBSTITUTE_SERVICE_TEMPLATE_PROPERTY_NAME); + } + /** * Add service template to map with key file name. |