From c181d7863fc1cf62d651e4ff09d9e52828b7f921 Mon Sep 17 00:00:00 2001 From: PriyanshuAgarwal Date: Tue, 6 Feb 2018 11:22:30 +0530 Subject: Import feature is ignoring multiple imports. Merged SDC-666 and SDC-668 as both are dependent. Change-Id: Idd4f67724d03bad79bab4a39b75a8145658ef8b9 Issue-ID: SDC-666 Signed-off-by: priyanshu --- .../sdc/toscaparser/api/ImportsLoader.java | 73 ++++--- .../sdc/toscaparser/api/ToscaTemplate.java | 212 +++++++++++++++++++-- .../sdc/toscaparser/api/JToscaImportTest.java | 64 +++++++ .../resources/csars/resource-Spgw-csar-ZTE.csar | Bin 0 -> 31639 bytes src/test/resources/csars/sdc-onboarding_csar.csar | Bin 0 -> 80596 bytes 5 files changed, 306 insertions(+), 43 deletions(-) create mode 100644 src/test/java/org/openecomp/sdc/toscaparser/api/JToscaImportTest.java create mode 100644 src/test/resources/csars/resource-Spgw-csar-ZTE.csar create mode 100644 src/test/resources/csars/sdc-onboarding_csar.csar diff --git a/src/main/java/org/openecomp/sdc/toscaparser/api/ImportsLoader.java b/src/main/java/org/openecomp/sdc/toscaparser/api/ImportsLoader.java index 5e94378..b2a0da7 100644 --- a/src/main/java/org/openecomp/sdc/toscaparser/api/ImportsLoader.java +++ b/src/main/java/org/openecomp/sdc/toscaparser/api/ImportsLoader.java @@ -28,6 +28,7 @@ public class ImportsLoader { private ArrayList typeDefinitionList; private LinkedHashMap customDefs; + private LinkedHashMap allCustomDefs; private ArrayList> nestedToscaTpls; private LinkedHashMap repositories; @@ -39,6 +40,7 @@ public class ImportsLoader { this.importslist = _importslist; customDefs = new LinkedHashMap(); + allCustomDefs = new LinkedHashMap(); nestedToscaTpls = new ArrayList>(); if((_path == null || _path.isEmpty()) && tpl == null) { //msg = _('Input tosca template is not provided.') @@ -65,7 +67,7 @@ public class ImportsLoader { } public LinkedHashMap getCustomDefs() { - return customDefs; + return allCustomDefs; } public ArrayList> getNestedToscaTpls() { @@ -131,33 +133,50 @@ public class ImportsLoader { } } - @SuppressWarnings("unchecked") + /** + * This method is used to get consolidated custom definitions by passing custom Types from + * each import. The resultant collection is then passed back which contains all import + * definitions + * + * @param customType the custom type + * @param namespacePrefix the namespace prefix + */ + @SuppressWarnings("unchecked") private void _updateCustomDefs(LinkedHashMap customType, String namespacePrefix) { - LinkedHashMap outerCustomTypes;// = new LinkedHashMap(); - for(String typeDef: typeDefinitionList) { - if(typeDef.equals("imports")) { - // imports are ArrayList... - customDefs.put("imports",(ArrayList)customType.get(typeDef)); - } - else { - outerCustomTypes = (LinkedHashMap)customType.get(typeDef); - if(outerCustomTypes != null) { - if(namespacePrefix != null && !namespacePrefix.isEmpty()) { - LinkedHashMap prefixCustomTypes = new LinkedHashMap(); - for(Map.Entry me: outerCustomTypes.entrySet()) { - String typeDefKey = me.getKey(); - String nameSpacePrefixToKey = namespacePrefix + "." + typeDefKey; - prefixCustomTypes.put(nameSpacePrefixToKey, outerCustomTypes.get(typeDefKey)); - } - customDefs.putAll(prefixCustomTypes); - } - else { - customDefs.putAll(outerCustomTypes); - } - } - } - } - } + LinkedHashMap outerCustomTypes; + for(String typeDef: typeDefinitionList) { + if(typeDef.equals("imports")) { + customDefs.put("imports", customType.get(typeDef)); + if (allCustomDefs.isEmpty() || allCustomDefs.get("imports") == null){ + allCustomDefs.put("imports",customType.get(typeDef)); + } + else if (customType.get(typeDef) != null){ + Set allCustomImports = new HashSet<>((ArrayList)allCustomDefs.get("imports")); + allCustomImports.addAll((ArrayList) customType.get(typeDef)); + allCustomDefs.put("imports", new ArrayList<>(allCustomImports)); + } + } + else { + outerCustomTypes = (LinkedHashMap)customType.get(typeDef); + if(outerCustomTypes != null) { + if(namespacePrefix != null && !namespacePrefix.isEmpty()) { + LinkedHashMap prefixCustomTypes = new LinkedHashMap(); + for(Map.Entry me: outerCustomTypes.entrySet()) { + String typeDefKey = me.getKey(); + String nameSpacePrefixToKey = namespacePrefix + "." + typeDefKey; + prefixCustomTypes.put(nameSpacePrefixToKey, outerCustomTypes.get(typeDefKey)); + } + customDefs.putAll(prefixCustomTypes); + allCustomDefs.putAll(prefixCustomTypes); + } + else { + customDefs.putAll(outerCustomTypes); + allCustomDefs.putAll(outerCustomTypes); + } + } + } + } + } private void _updateNestedToscaTpls(String fullFileName,LinkedHashMap customTpl) { if(fullFileName != null && customTpl != null) { diff --git a/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java b/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java index 4c19be6..e96ca56 100644 --- a/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java +++ b/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java @@ -9,6 +9,9 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.nio.file.Files; +import java.util.function.Predicate; +import java.nio.file.Paths; import org.openecomp.sdc.toscaparser.api.common.ValidationIssueCollector; import org.openecomp.sdc.toscaparser.api.common.JToscaException; @@ -70,6 +73,7 @@ public class ToscaTemplate extends Object { private boolean isFile; private String path; private String inputPath; + private String rootPath; private LinkedHashMap parsedParams; private boolean resolveGetInput; private LinkedHashMap tpl; @@ -91,6 +95,7 @@ public class ToscaTemplate extends Object { private String csarTempDir; private int nestingLoopCounter; private LinkedHashMap> metaProperties; + private Set processedImports; public ToscaTemplate(String _path, LinkedHashMap _parsedParams, @@ -193,6 +198,9 @@ public class ToscaTemplate extends Object { if(tpl != null) { parsedParams = _parsedParams; _validateField(); + this.rootPath = path; + this.processedImports = new HashSet(); + this.imports = _tplImports(); this.version = _tplVersion(); this.metaData = _tplMetaData(); this.relationshipTypes = _tplRelationshipTypes(); @@ -305,30 +313,200 @@ public class ToscaTemplate extends Object { private ArrayList _policies() { return topologyTemplate.getPolicies(); } - - private LinkedHashMap _getAllCustomDefs(ArrayList alImports) { - + + /** + * This method is used to get consolidated custom definitions from all imports + * It is logically divided in two parts to handle imports; map and list formats. + * Before processing the imports; it sorts them to make sure the current directory imports are + * being processed first and then others. Once sorted; it processes each import one by one in + * recursive manner. + * To avoid cyclic dependency among imports; this method uses a set to keep track of all + * imports which are already processed and filters the imports which occurs more than once. + * + * @param alImports all imports which needs to be processed + * @return the linked hash map containing all import definitions + */ + private LinkedHashMap _getAllCustomDefs(Object alImports) { + String types[] = { - IMPORTS, NODE_TYPES, CAPABILITY_TYPES, RELATIONSHIP_TYPES, - DATA_TYPES, INTERFACE_TYPES, POLICY_TYPES, GROUP_TYPES + IMPORTS, NODE_TYPES, CAPABILITY_TYPES, RELATIONSHIP_TYPES, + DATA_TYPES, INTERFACE_TYPES, POLICY_TYPES, GROUP_TYPES }; - LinkedHashMap customDefsFinal = new LinkedHashMap(); - LinkedHashMap customDefs = _getCustomTypes(types,alImports); - if(customDefs != null) { - customDefsFinal.putAll(customDefs); - if(customDefs.get(IMPORTS) != null) { - @SuppressWarnings("unchecked") - LinkedHashMap importDefs = _getAllCustomDefs((ArrayList)customDefs.get(IMPORTS)); - customDefsFinal.putAll(importDefs); + LinkedHashMap customDefsFinal = new LinkedHashMap<>(); + + List> imports = (List>) alImports; + if (imports != null && !imports.isEmpty()) { + if (imports.get(0) instanceof LinkedHashMap) { + imports = sortImports(imports); + + for (Map map : imports) { + List> singleImportList = new ArrayList(); + singleImportList.add(map); + + Map importNameDetails = getValidFileNameForImportReference(singleImportList); + singleImportList = filterImportsForRecursion(singleImportList, importNameDetails); + + if(!singleImportList.get(0).isEmpty()){ + LinkedHashMap customDefs = _getCustomTypes(types, new ArrayList<>(singleImportList)); + processedImports.add(importNameDetails.get("importFileName")); + + if (customDefs != null) { + customDefsFinal.putAll(customDefs); + + if (customDefs.get(IMPORTS) != null) { + resetPathForRecursiveImports(importNameDetails.get("importRelativeName")); + LinkedHashMap importDefs = _getAllCustomDefs(customDefs.get(IMPORTS)); + customDefsFinal.putAll(importDefs); + } + } + } + } + } else { + LinkedHashMap customDefs = _getCustomTypes(types, new ArrayList<>(imports)); + if (customDefs != null) { + customDefsFinal.putAll(customDefs); + + if (customDefs.get(IMPORTS) != null) { + LinkedHashMap importDefs = _getAllCustomDefs(customDefs.get(IMPORTS)); + customDefsFinal.putAll(importDefs); + } + } } } - - // As imports are not custom_types, remove from the dict - customDefsFinal.remove(IMPORTS); + + // As imports are not custom_types, remove from the dict + customDefsFinal.remove(IMPORTS); return customDefsFinal; } + /** + * This method is used to sort the imports in order so that same directory + * imports will be processed first + * + * @param customImports the custom imports + * @return the sorted list of imports + */ + private List> sortImports(List> customImports){ + List> finalList1 = new ArrayList<>(); + List> finalList2 = new ArrayList<>(); + Iterator> itr = customImports.iterator(); + while(itr.hasNext()) { + Map innerMap = itr.next(); + if (innerMap.toString().contains("../")) { + finalList2.add(innerMap); + itr.remove(); + } + else if (innerMap.toString().contains("/")) { + finalList1.add(innerMap); + itr.remove(); + } + } + + customImports.addAll(finalList1); + customImports.addAll(finalList2); + return customImports; + } + + /** + * This method is used to reset PATH variable after processing of current import file is done + * This is required because of relative path nature of imports present in files. + * + * @param currImportRelativeName the current import relative name + */ + private void resetPathForRecursiveImports(String currImportRelativeName){ + path = getPath(path, currImportRelativeName); + } + + /** + * This is a recursive method which starts from current import and then recursively finds a + * valid path relative to current import file name. + * By doing this it handles all nested hierarchy of imports defined in CSARs + * + * @param path the path + * @param importFileName the import file name + * @return the string containing updated path value + */ + private String getPath(String path, String importFileName){ + String tempFullPath = (Paths.get(path).toAbsolutePath().getParent() + .toString() + File.separator + importFileName.replace("../", "")).replace('\\', '/'); + String tempPartialPath = (Paths.get(path).toAbsolutePath().getParent().toString()).replace('\\', '/'); + if(Files.exists(Paths.get(tempFullPath))) + return tempFullPath; + else + return getPath(tempPartialPath, importFileName); + } + + /** + * This method is used to get full path name for the file which needs to be processed. It helps + * in situation where files are present in different directory and are references as relative + * paths. + * + * @param customImports the custom imports + * @return the map containing import file full and relative paths + */ + private Map getValidFileNameForImportReference(List> + customImports){ + String importFileName; + Map retMap = new HashMap<>(); + for (Map map1 : customImports) { + for (Map.Entry entry : map1.entrySet()) { + Map innerMostMap = (Map) entry.getValue(); + Iterator> it = innerMostMap.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry val = it.next(); + if(val.getValue().contains("/")){ + importFileName = (Paths.get(rootPath).toAbsolutePath().getParent().toString() + File + .separator + val.getValue().replace("../", "")).replace('\\', '/'); + } + else { + importFileName = (Paths.get(path).toAbsolutePath().getParent().toString() + File + .separator + val.getValue().replace("../", "")).replace('\\', '/'); + } + retMap.put("importFileName", importFileName); + retMap.put("importRelativeName", val.getValue()); + } + } + } + return retMap; + } + + /** + * This method is used to filter the imports which already gets processed in previous step. + * It handles the use case of cyclic dependency in imports which may cause Stack Overflow + * exception + * + * @param customImports the custom imports + * @param importNameDetails the import name details + * @return the list containing filtered imports + */ + private List> filterImportsForRecursion(List> + customImports, Map importNameDetails){ + for (Map map1 : customImports) { + for (Map.Entry entry : map1.entrySet()) { + Map innerMostMap = (Map) entry.getValue(); + Iterator> it = innerMostMap.entrySet().iterator(); + while (it.hasNext()) { + it.next(); + if (processedImports.contains(importNameDetails.get("importFileName"))) { + it.remove(); + } + } + } + } + + // Remove Empty elements + Iterator> itr = customImports.iterator(); + while(itr.hasNext()) { + Map innerMap = itr.next(); + Predicate predicate = p-> p.values().isEmpty(); + innerMap.values().removeIf(predicate); + } + + return customImports; + } + @SuppressWarnings("unchecked") private LinkedHashMap _getCustomTypes(Object typeDefinitions,ArrayList alImports) { @@ -396,6 +574,8 @@ public class ToscaTemplate extends Object { log.error("ToscaTemplate - _handleNestedToscaTemplatesWithTopology - Nested Topologies Loop: too many levels, aborting"); return; } + // Reset Processed Imports for nested templates + this.processedImports = new HashSet<>(); for(Map.Entry me: nestedToscaTplsWithTopology.entrySet()) { String fname = me.getKey(); LinkedHashMap toscaTpl = diff --git a/src/test/java/org/openecomp/sdc/toscaparser/api/JToscaImportTest.java b/src/test/java/org/openecomp/sdc/toscaparser/api/JToscaImportTest.java new file mode 100644 index 0000000..c8a30fa --- /dev/null +++ b/src/test/java/org/openecomp/sdc/toscaparser/api/JToscaImportTest.java @@ -0,0 +1,64 @@ +package org.openecomp.sdc.toscaparser.api; + +import org.junit.Test; +import org.openecomp.sdc.toscaparser.api.common.JToscaException; +import org.openecomp.sdc.toscaparser.api.utils.ThreadLocalsHolder; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; + +public class JToscaImportTest { + + @Test + public void testNoMissingTypeValidationError() throws JToscaException { + String fileStr = JToscaImportTest.class.getClassLoader().getResource + ("csars/sdc-onboarding_csar.csar").getFile(); + File file = new File(fileStr); + new ToscaTemplate(file.getAbsolutePath(), null, true, null); + List missingTypeErrors = ThreadLocalsHolder.getCollector() + .getValidationIssueReport() + .stream() + .filter(s -> s.contains("JE136")) + .collect(Collectors.toList()); + assertEquals(0, missingTypeErrors.size()); + } + + @Test + public void testNoStackOverFlowError() { + Exception jte = null; + try { + String fileStr = JToscaImportTest.class.getClassLoader().getResource + ("csars/sdc-onboarding_csar.csar").getFile(); + File file = new File(fileStr); + new ToscaTemplate(file.getAbsolutePath(), null, true, null); + } catch (Exception e){ + jte = e; + } + assertEquals(null, jte); + } + + @Test + public void testNoInvalidImports() throws JToscaException { + List fileNames = new ArrayList<>(); + fileNames.add("csars/tmpCSAR_Huawei_vSPGW_fixed.csar"); + fileNames.add("csars/sdc-onboarding_csar.csar"); + fileNames.add("csars/resource-Spgw-csar-ZTE.csar"); + + for (String fileName : fileNames) { + String fileStr = JToscaImportTest.class.getClassLoader().getResource(fileName).getFile(); + File file = new File(fileStr); + new ToscaTemplate(file.getAbsolutePath(), null, true, null); + List invalidImportErrors = ThreadLocalsHolder.getCollector() + .getValidationIssueReport() + .stream() + .filter(s -> s.contains("JE195")) + .collect(Collectors.toList()); + assertEquals(0, invalidImportErrors.size()); + } + } + +} diff --git a/src/test/resources/csars/resource-Spgw-csar-ZTE.csar b/src/test/resources/csars/resource-Spgw-csar-ZTE.csar new file mode 100644 index 0000000..58c3ddd Binary files /dev/null and b/src/test/resources/csars/resource-Spgw-csar-ZTE.csar differ diff --git a/src/test/resources/csars/sdc-onboarding_csar.csar b/src/test/resources/csars/sdc-onboarding_csar.csar new file mode 100644 index 0000000..e1c3267 Binary files /dev/null and b/src/test/resources/csars/sdc-onboarding_csar.csar differ -- cgit 1.2.3-korg