From bf5eeb23a769a2e2b75f432b74f10fdbcfd2f161 Mon Sep 17 00:00:00 2001 From: "andre.schmid" Date: Fri, 27 Sep 2019 13:27:11 +0100 Subject: Fix zip slip security flaw Apply zip slip checking in zip operations throughout the system. Centralizes most of the zip logic in one class. Create tests to zip functionalities and zip slip problem. Change-Id: I721f3d44b34fe6d242c9537f5a515ce1bb534c9a Issue-ID: SDC-1401 Signed-off-by: andre.schmid --- .../sdc/ci/tests/utils/ToscaParserUtils.java | 289 +++------------------ .../sdc/ci/tests/utils/general/FileHandling.java | 148 ++--------- .../sdc/externalApis/DeploymentValiditaion.java | 30 --- 3 files changed, 54 insertions(+), 413 deletions(-) (limited to 'test-apis-ci/src/main/java/org/openecomp') diff --git a/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/ToscaParserUtils.java b/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/ToscaParserUtils.java index be235d03ae..b7eaf99724 100644 --- a/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/ToscaParserUtils.java +++ b/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/ToscaParserUtils.java @@ -20,29 +20,40 @@ package org.openecomp.sdc.ci.tests.utils; +import static org.testng.AssertJUnit.assertNotNull; +import static org.testng.AssertJUnit.assertTrue; +import static org.testng.AssertJUnit.fail; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import org.apache.commons.collections4.MapUtils; import org.openecomp.sdc.ci.tests.datatypes.enums.UserRoleEnum; -import org.openecomp.sdc.ci.tests.tosca.datatypes.*; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaGroupsTopologyTemplateDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaImportsDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaInputsTopologyTemplateDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaMetadataDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaNodeTemplatesTopologyTemplateDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaParameterConstants; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaSubstitutionMappingsDefinition; +import org.openecomp.sdc.ci.tests.tosca.datatypes.ToscaTopologyTemplateDefinition; import org.openecomp.sdc.ci.tests.utils.general.ElementFactory; import org.openecomp.sdc.ci.tests.utils.rest.BaseRestUtils; import org.openecomp.sdc.ci.tests.utils.rest.ImportRestUtils; import org.openecomp.sdc.common.http.client.api.HttpResponse; -import org.openecomp.sdc.common.util.ZipUtil; +import org.openecomp.sdc.common.zip.ZipUtils; +import org.openecomp.sdc.common.zip.exception.ZipException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; import org.yaml.snakeyaml.introspector.PropertyUtils; -import java.io.File; -import java.io.FileInputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; - -import static org.testng.AssertJUnit.assertNotNull; -import static org.testng.AssertJUnit.assertTrue; - public class ToscaParserUtils { private static Logger log = LoggerFactory.getLogger(ToscaParserUtils.class.getName()); @@ -135,10 +146,8 @@ public class ToscaParserUtils { try { toscaDefinition = (ToscaDefinition) yaml.load(payload); } catch (Exception e) { - log.debug("Failed to parse tosca yaml file"); - log.debug("Exception: " + e); - System.out.println("Exception: " + e); - assertTrue("Exception: " + e, false); + log.debug("Failed to parse tosca yaml file", e); + fail("Exception: " + e); } return toscaDefinition; @@ -155,7 +164,6 @@ public class ToscaParserUtils { toscaStructure.addTypeDescription(ToscaImportsDefinition.getTypeDescription()); toscaStructure.addTypeDescription(ToscaMetadataDefinition.getTypeDescription()); toscaStructure.addTypeDescription(ToscaInputsTopologyTemplateDefinition.getTypeDescription()); -// toscaStructure.addTypeDescription(ToscaInputsDefinition.getTypeDescription()); // Skip properties which are found in YAML, but not found in POJO PropertyUtils propertyUtils = new PropertyUtils(); propertyUtils.setSkipMissingProperties(true); @@ -193,12 +201,10 @@ public class ToscaParserUtils { } - public static String getYamlPayloadFromCsar(File csarName, String fileLocation) throws Exception { - + public static String getYamlPayloadFromCsar(File csarName, String fileLocation) throws IOException, ZipException { Path path = csarName.toPath(); byte[] data = Files.readAllBytes(path); return getDataFromZipFileByBytes(fileLocation, data); - } /** method get file data from zip data by file location in the zip structure @@ -206,243 +212,16 @@ public class ToscaParserUtils { * @param data * @return */ - public static String getDataFromZipFileByBytes(String fileLocation, byte[] data) { - Map readZip = null; - if (data != null && data.length > 0) { - readZip = ZipUtil.readZip(data); - + public static String getDataFromZipFileByBytes(String fileLocation, byte[] data) throws ZipException { + if (data == null || data.length == 0) { + return null; } - byte[] artifactsBs = readZip.get(fileLocation); - String str = new String(artifactsBs, StandardCharsets.UTF_8); - return str; - } -/* public static Map getToscaYamlMap(String csarUUID, String fileLocation) throws Exception { - String csarPayload = CsarValidationUtils.getCsarPayload(csarUUID, fileLocation); - if (csarPayload != null) { - Yaml yaml = new Yaml(); - Map map = (Map) yaml.load(csarPayload); - return map; + final Map readZip = ZipUtils.readZip(data, false); + if (MapUtils.isEmpty(readZip)) { + return null; } - return null; + byte[] artifactsBytes = readZip.get(fileLocation); + return new String(artifactsBytes, StandardCharsets.UTF_8); } - - public static ToscaDefinition getToscaDefinitionObjectByCsarUuid(String csarUUID) throws Exception { - - String TOSCAMetaLocation = "TOSCA-Metadata/TOSCA.meta"; - Map map = getToscaYamlMap(csarUUID, TOSCAMetaLocation); - assertNotNull("Tosca Entry-Definitions is null", map); - if (map != null) { - String definitionYamlLocation = (String) map.get("Entry-Definitions"); - Map toscaMap = getToscaYamlMap(csarUUID, definitionYamlLocation); - assertNotNull("Tosca definition is null", toscaMap); - if (toscaMap != null) { - ToscaDefinition toscaDefinition = new ToscaDefinition(); - Set keySet = toscaMap.keySet(); - for (Object key : keySet) { - ToscaKeysEnum toscaKey = ToscaKeysEnum.findToscaKey((String) key); - switch (toscaKey) { - case TOSCA_DEFINITION_VERSION: - getToscaDefinitionVersion(toscaMap, toscaDefinition); - break; - case NODE_TYPES: - getToscaNodeTypes(toscaMap, toscaDefinition); - break; - case TOPOLOGY_TEMPLATE: - getToscaTopologyTemplate(toscaMap, toscaDefinition); - break; - case IMPORTS: - // toscaMap.get("imports"); - break; - default: - break; - } - } - return toscaDefinition; - } - } - return null; - - } - - public static void getToscaDefinitionVersion(Map toscaMap, ToscaDefinition toscaDefinition) { - if (toscaMap.get("tosca_definitions_version") != null) { - toscaDefinition.setTosca_definitions_version((String) toscaMap.get("tosca_definitions_version")); - } - } - - // spec 90 page - public static void getToscaNodeTypes(Map toscaMap, ToscaDefinition toscaDefinition) { - @SuppressWarnings("unchecked") - Map> nodeTypes = (Map>) toscaMap.get("node_types"); - Map listToscaNodeTypes = new HashMap(); - if (nodeTypes != null) { - for (Map.Entry> entry : nodeTypes.entrySet()) { - ToscaNodeTypesDefinition toscaNodeTypes = new ToscaNodeTypesDefinition(); - String toscaNodeName = entry.getKey(); - toscaNodeTypes.setName(toscaNodeName); - - Map toscaNodeType = entry.getValue(); - if (toscaNodeType != null) { - Set> entrySet = toscaNodeType.entrySet(); - if (entrySet != null) { - // boolean found = false; - for (Entry toscaNodeTypeMap : entrySet) { - String key = toscaNodeTypeMap.getKey(); - if (key.equals("derived_from")) { - String derivedFrom = toscaNodeTypeMap.getValue(); - toscaNodeTypes.setDerived_from(derivedFrom); - // found = true; - break; - } else { - continue; - } - - } - // if (found == false) { - // System.out.println("Tosca file not valid, - // derived_from not found"); - // } - } - - } -// listToscaNodeTypes.add(toscaNodeTypes); - listToscaNodeTypes.put(toscaNodeName, toscaNodeTypes); - } - toscaDefinition.setNode_types(listToscaNodeTypes); - } - } - - public static void getToscaTopologyTemplate(Map toscaMap, ToscaDefinition toscaDefinition) { - ToscaTopologyTemplateDefinition toscaTopologyTemplate = new ToscaTopologyTemplateDefinition(); - @SuppressWarnings("unchecked") - Map> topologyTemplateMap = (Map>) toscaMap.get("topology_template"); -// List listToscaNodeTemplates = new ArrayList<>(); - Map mapToscaNodeTemplates = new HashMap(); - - if (topologyTemplateMap != null) { - getToscaNodeTemplates(topologyTemplateMap, mapToscaNodeTemplates); - } -// toscaTopologyTemplate.setToscaNodeTemplatesTopologyTemplateDefinition(listToscaNodeTemplates); - toscaTopologyTemplate.setNode_templates(mapToscaNodeTemplates); - toscaDefinition.setTopology_template(toscaTopologyTemplate); - } - - public static void getToscaNodeTemplates(Map> topologyTemplateMap, Map mapToscaNodeTemplates) { - Map nodeTemplatesMap = topologyTemplateMap.get("node_templates"); - if (nodeTemplatesMap != null) { - - for (Entry nodeTemplates : nodeTemplatesMap.entrySet()) { - ToscaNodeTemplatesTopologyTemplateDefinition toscaNodeTemplates = new ToscaNodeTemplatesTopologyTemplateDefinition(); - getToscaNodeTemplatesName(nodeTemplates, toscaNodeTemplates); - - @SuppressWarnings("unchecked") - Map node = (Map) nodeTemplates.getValue(); - getNodeTemplatesType(toscaNodeTemplates, node); - getToscaNodeTemplateProperties(toscaNodeTemplates, node); - getToscaNodeTemplateRequirements(toscaNodeTemplates, node); - mapToscaNodeTemplates.putAll(mapToscaNodeTemplates); - } - } - } - - public static void getToscaNodeTemplateRequirements(ToscaNodeTemplatesTopologyTemplateDefinition toscaNodeTemplates, Map node) { -//// List toscaRequirements = new ArrayList<>(); -// List> toscaRequirements = new ArrayList<>(); -// if (node.get("requirements") != null) { -// @SuppressWarnings("unchecked") -// List> requirementList = (List>) node.get("requirements"); -// for (int i = 0; i < requirementList.size(); i++) { -// for (Map.Entry requirement : requirementList.get(i).entrySet()) { -// ToscaRequirementsNodeTemplatesDefinition toscaRequirement = new ToscaRequirementsNodeTemplatesDefinition(); -// if (requirement.getKey() != null) { -// String requirementName = requirement.getKey(); -// toscaRequirement.setName(requirementName); -// } else { -// log.debug("Tosca file not valid, requirements should contain name"); -// } -// -// @SuppressWarnings("unchecked") -// Map requirementMap = (Map) requirement.getValue(); -// Set> entrySet = requirementMap.entrySet(); -// if (entrySet != null) { -// for (Entry requirementField : entrySet) { -// String key = requirementField.getKey(); -// switch (key) { -// case "capability": -// if (requirementMap.get(key) != null) { -// String capability = (String) requirementMap.get(key); -// toscaRequirement.setCapability(capability); -// break; -// } else { -// continue; -// } -// case "node": -// if (requirementMap.get(key) != null) { -// String requirementNode = (String) requirementMap.get(key); -// toscaRequirement.setNode(requirementNode); -// break; -// } else { -// continue; -// } -// case "relationship": -// if (requirementMap.get(key) != null) { -// String relationship = (String) requirementMap.get(key); -// toscaRequirement.setRelationship(relationship); -// break; -// } else { -// continue; -// } -// default: -// break; -// } -// } -// } -//// toscaRequirements.add(toscaRequirement); -// toscaRequirements.add(requirementMap); -// } -// } -// } -//// toscaNodeTemplates.setRequirements(toscaRequirements); -// toscaNodeTemplates.setRequirements(requirements); - - } - - public static void getToscaNodeTemplateProperties(ToscaNodeTemplatesTopologyTemplateDefinition toscaNodeTemplates, - Map node) { -// List listToscaProperties = new ArrayList<>(); - Map mapToscaProperties = new HashMap<>(); - if (node.get("properties") != null) { - @SuppressWarnings("unchecked") - Map properties = (Map) node.get("properties"); - for (Map.Entry property : properties.entrySet()) { - ToscaPropertiesNodeTemplatesDefinition toscaProperty = new ToscaPropertiesNodeTemplatesDefinition(); - String propertyName = property.getKey(); - Object propertyValue = property.getValue(); - toscaProperty.setName(propertyName); - toscaProperty.setValue(propertyValue); -// mapToscaProperties.add(toscaProperty); - mapToscaProperties.put(propertyName, propertyValue); - } - } - toscaNodeTemplates.setProperties(mapToscaProperties); - } - - protected static void getNodeTemplatesType(ToscaNodeTemplatesTopologyTemplateDefinition toscaNodeTemplates, - Map node) { - if (node.get("type") != null) { - String type = (String) node.get("type"); - toscaNodeTemplates.setType(type); - } else { - log.debug("Tosca file not valid, nodeTemplate should contain type"); - } - } - - protected static void getToscaNodeTemplatesName(Entry nodeTemplates, - ToscaNodeTemplatesTopologyTemplateDefinition toscaNodeTemplates) { - String name = nodeTemplates.getKey(); - toscaNodeTemplates.setName(name); - }*/ - - } diff --git a/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/general/FileHandling.java b/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/general/FileHandling.java index 4dbc8b2dee..dae13a2c82 100644 --- a/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/general/FileHandling.java +++ b/test-apis-ci/src/main/java/org/openecomp/sdc/ci/tests/utils/general/FileHandling.java @@ -20,25 +20,33 @@ package org.openecomp.sdc.ci.tests.utils.general; +import static org.testng.AssertJUnit.assertTrue; + import com.aventstack.extentreports.Status; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; import org.apache.commons.io.FileUtils; import org.openecomp.sdc.be.model.DataTypeDefinition; import org.openecomp.sdc.ci.tests.api.ComponentBaseTest; -import org.openecomp.sdc.ci.tests.config.Config; import org.openecomp.sdc.ci.tests.datatypes.enums.XnfTypeEnum; import org.openecomp.sdc.common.util.GeneralUtility; import org.yaml.snakeyaml.Yaml; -import java.io.*; -import java.nio.file.Paths; -import java.util.*; -import java.util.zip.ZipEntry; -import java.util.zip.ZipException; -import java.util.zip.ZipFile; -import java.util.zip.ZipInputStream; - -import static org.testng.AssertJUnit.assertTrue; - public class FileHandling { // ------------------yaml parser methods---------------------------- @@ -264,10 +272,6 @@ public class FileHandling { return null; } -// public static Object[] getZipFileNamesFromFolder(String filepath) { -// return filterFileNamesFromFolder(filepath, ".zip"); -// } - public static List getZipFileNamesFromFolder(String filepath) { List fileNamesListFromFolder = filterFileNamesListFromFolder(filepath, ".zip"); fileNamesListFromFolder.addAll(filterFileNamesListFromFolder(filepath, ".csar")); @@ -285,16 +289,6 @@ public class FileHandling { return fileCounter; } - - /** - * @return last modified file name from default directory - * @throws Exception - */ -// TODO add download directory capability -// public static synchronized File getLastModifiedFileNameFromDir() throws Exception{ -// return getLastModifiedFileNameFromDir(ComponentBaseTest.getWindowTest().getDownloadDirectory()); -// } - /** * @param dirPath * @return last modified file name from dirPath directory @@ -369,52 +363,6 @@ public class FileHandling { ComponentBaseTest.getExtendTest().log(Status.INFO, "Unable to write to flie " + pathToFile); } } - -// public static synchronized void writeToFile(File pathToFile, Map> dataMap, Integer leftSpaceCount) throws IOException{ -// -// BufferedWriter bw = null; -// FileWriter fw = null; -// try { -// if(!pathToFile.exists()){ -// createEmptyFile(pathToFile); -// } -// fw = new FileWriter(pathToFile, true); -// bw = new BufferedWriter(fw); -// StringBuilder sb = new StringBuilder(); -// if(leftSpaceCount > 0 ){ -// for(int i = 0; i < leftSpaceCount; i++){ -// sb.append(" "); -// } -// } -// for(Map.Entry> entry : dataMap.entrySet()){ -// Object record = ArtifactUIUtils.getFormatedData(entry.getKey(), entry.getValue().right); -// bw.write(sb.toString() + record); -// bw.newLine(); -// } -// bw.close(); -// fw.close(); -// } catch (Exception e) { -// ComponentBaseTest.getExtendTest().log(Status.INFO, "Unable to write to flie " + pathToFile); -// } -// } - - public static void deleteLastDowloadedFiles(List files) throws IOException { - for (File file : files) { - File fileToDelete =new File(Config.instance().getWindowsDownloadDirectory()+file.getName()); - fileToDelete.delete(); - } - } -// TODO add work with directory capaability -// public static void cleanCurrentDownloadDir() throws IOException { -// try{ -// ExtentTestActions.log(Status.INFO, "Cleaning directory " + ComponentBaseTest.getWindowTest().getDownloadDirectory()); -// System.gc(); -// FileUtils.cleanDirectory(new File(ComponentBaseTest.getWindowTest().getDownloadDirectory())); -// } -// catch(Exception e){ -// -// } -// } public static String getCreateDirByName(String dirName) { File dir = new File(dirName); @@ -507,63 +455,7 @@ public class FileHandling { } return fileList; } - - private static final int BUFFER_SIZE = 4096; -// public static void unzip(String zipFilePath, String destDirectory) throws IOException { -// File destDir = new File(destDirectory); -// if (!destDir.exists()) { -// destDir.mkdir(); -// } -// ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFilePath)); -// ZipEntry entry = zipIn.getNextEntry(); -//// iterates over entries in the zip file -// while (entry != null) { -// String entryName; -// if(System.getProperty("os.name").contains("Windows")){ -// entryName = entry.getName().replaceAll("/", "\\"+File.separator); -// }else{ -// entryName = entry.getName(); -// } -// String filePath = destDirectory + entryName; -// String currPath = destDirectory; -// String[] dirs = entryName.split("\\"+File.separator); -// String currToken; -// for(int i = 0; i readZip = null; - Path path = Paths.get("C:\\Users\\ys9693\\Documents\\csar\\attributesWithProporties\\attributesWithProporties.csar"); - byte[] data = Files.readAllBytes(path); - if (data != null && data.length > 0) { - readZip = ZipUtil.readZip(data); - - } - - byte[] artifactsBs = readZip.get("Definitions/VF_RI2_G6.yaml"); - String str = new String(artifactsBs, StandardCharsets.UTF_8); - - Yaml yaml = new Yaml(); - Map load = (Map) yaml.load(str); - Map topology_template = (Map) load.get("topology_template"); - Map node_templates = (Map) topology_template.get("node_templates"); - - Set keySet = node_templates.keySet(); - } - - - @Test public void pasrseDataTypes() throws Exception{ -- cgit 1.2.3-korg