From af2d8ef12f6b75ae837fffdeaf5d152756cec6d6 Mon Sep 17 00:00:00 2001 From: Pavel Aharoni Date: Thu, 11 May 2017 16:41:38 +0300 Subject: [SDC-18] Tosca Parser conformance level - initial Change-Id: I0d8d27f4b8085d918ecc94aace423d3216337f2d Signed-off-by: Pavel Aharoni --- .gitignore | 2 + jtosca/pom.xml | 9 +- .../sdc/toscaparser/api/ToscaTemplate.java | 10 +- .../openecomp/sdc/toscaparser/api/prereq/CSAR.java | 176 +++++++++++---------- .../JToscaMetadataParse.java | 26 +++ .../src/test/resources/csars/csar_hello_world.csar | Bin 0 -> 936 bytes pom.xml | 2 +- sdc-distribution-ci/pom.xml | 2 +- sdc-distribution-client/pom.xml | 2 +- sdc-tosca-parser/pom.xml | 21 ++- .../sdc/tosca/parser/api/ConformanceLevel.java | 25 +++ .../sdc/tosca/parser/api/ISdcCsarHelper.java | 4 +- .../openecomp/sdc/tosca/parser/api/Version.java | 48 ++++++ .../sdc/tosca/parser/config/Configuration.java | 16 ++ .../tosca/parser/config/ConfigurationManager.java | 51 ++++++ .../sdc/tosca/parser/impl/SdcCsarHelperImpl.java | 19 +++ .../tosca/parser/impl/SdcToscaParserFactory.java | 105 +++++++----- .../sdc/tosca/parser/utils/GeneralUtility.java | 53 ++++++- .../tosca/parser/utils/YamlToObjectConverter.java | 123 ++++++++++++++ .../src/main/resources/config/configuration.yaml | 3 + .../java/org/openecomp/sdc/impl/BasicTest.java | 134 ++++++++++++++-- .../sdc/impl/ToscaParserConfigurationTest.java | 21 +++ .../sdc/impl/ToscaParserGeneralUtilTest.java | 23 +++ .../openecomp/sdc/impl/ToscaParserGroupTest.java | 28 ++-- .../sdc/impl/ToscaParserMetadataTest.java | 35 ++-- .../sdc/impl/ToscaParserNodeTemplateTest.java | 118 +++++++------- .../sdc/impl/ToscaParserServiceInputTest.java | 18 +-- .../sdc/impl/ToscaParserSubsMappingsTest.java | 118 +++++++++++++- .../openecomp/sdc/impl/ToscaParserTestSuite.java | 56 ------- .../src/test/resources/config/configuration.yaml | 3 + .../src/test/resources/csars/csar_hello_world.zip | Bin 936 -> 0 bytes .../csars/service-ServiceFdnt-csar-0904-2.csar | Bin 41245 -> 41437 bytes .../csars/service-ServiceFdnt-csar-rainy.csar | Bin 39700 -> 39892 bytes .../resources/csars/service-ServiceFdnt-csar.csar | Bin 39979 -> 40171 bytes .../csars/service-ServiceFdnt-with-allotted.csar | Bin 44976 -> 45194 bytes 35 files changed, 943 insertions(+), 308 deletions(-) create mode 100644 jtosca/src/test/java/org.openecomp.sdc.toscaparser/JToscaMetadataParse.java create mode 100644 jtosca/src/test/resources/csars/csar_hello_world.csar create mode 100644 sdc-tosca-parser/src/main/java/org/openecomp/sdc/tosca/parser/api/ConformanceLevel.java create mode 100644 sdc-tosca-parser/src/main/java/org/openecomp/sdc/tosca/parser/api/Version.java create mode 100644 sdc-tosca-parser/src/main/java/org/openecomp/sdc/tosca/parser/config/Configuration.java create mode 100644 sdc-tosca-parser/src/main/java/org/openecomp/sdc/tosca/parser/config/ConfigurationManager.java create mode 100644 sdc-tosca-parser/src/main/java/org/openecomp/sdc/tosca/parser/utils/YamlToObjectConverter.java create mode 100644 sdc-tosca-parser/src/main/resources/config/configuration.yaml create mode 100644 sdc-tosca-parser/src/test/java/org/openecomp/sdc/impl/ToscaParserConfigurationTest.java create mode 100644 sdc-tosca-parser/src/test/java/org/openecomp/sdc/impl/ToscaParserGeneralUtilTest.java delete mode 100644 sdc-tosca-parser/src/test/java/org/openecomp/sdc/impl/ToscaParserTestSuite.java create mode 100644 sdc-tosca-parser/src/test/resources/config/configuration.yaml delete mode 100644 sdc-tosca-parser/src/test/resources/csars/csar_hello_world.zip diff --git a/.gitignore b/.gitignore index 68f22e6..74f7cda 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ target/ *.class *.orig .idea/* + +sdc-tosca-parser/test-output/**/* diff --git a/jtosca/pom.xml b/jtosca/pom.xml index f74988b..ce50fec 100644 --- a/jtosca/pom.xml +++ b/jtosca/pom.xml @@ -9,7 +9,7 @@ jtosca - 0.1.2-SNAPSHOT + 0.1.3-SNAPSHOT @@ -34,5 +34,10 @@ test --> - + + junit + junit + 4.12 + + \ No newline at end of file diff --git a/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java b/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java index 942ca96..d1b0179 100644 --- a/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java +++ b/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/ToscaTemplate.java @@ -82,8 +82,9 @@ public class ToscaTemplate extends Object { private ArrayList nestedToscaTemplatesWithTopology; private ToscaGraph graph; private String csarTempDir; - private int nestingLoopCounter; - + private int nestingLoopCounter; + private LinkedHashMap> metaProperties; + @SuppressWarnings("unchecked") public ToscaTemplate(String _path, LinkedHashMap _parsedParams, @@ -480,6 +481,7 @@ public class ToscaTemplate extends Object { if (csar.validate()) { try { csar.decompress(); + metaProperties = csar.getMetaProperties(); } catch (IOException e) { log.error("ToscaTemplate - _getPath - IOException trying to decompress {}", _path); @@ -554,6 +556,10 @@ public class ToscaTemplate extends Object { public ArrayList getNodeTemplates() { return nodeTemplates; } + + public LinkedHashMap getMetaProperties(String propertiesFile) { + return metaProperties.get(propertiesFile); + } private boolean _isSubMappedNode(NodeTemplate nt,LinkedHashMap toscaTpl) { // Return True if the nodetemple is substituted diff --git a/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/prereq/CSAR.java b/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/prereq/CSAR.java index 159ad66..2ab5d26 100644 --- a/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/prereq/CSAR.java +++ b/jtosca/src/main/java/org/openecomp/sdc/toscaparser/api/prereq/CSAR.java @@ -12,10 +12,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipInputStream; @@ -31,6 +28,7 @@ import org.yaml.snakeyaml.Yaml; public class CSAR { private static Logger log = LoggerFactory.getLogger(CSAR.class.getName()); + private static final ArrayList META_PROPERTIES_FILES = new ArrayList<>(Arrays.asList("TOSCA-Metadata/TOSCA.meta", "csar.meta")); private String path; private boolean isFile; @@ -38,9 +36,10 @@ public class CSAR { private boolean errorCaught; private String csar; private String tempDir; - private Metadata metaData; +// private Metadata metaData; private File tempFile; - + private LinkedHashMap> metaProperties; + public CSAR(String csarPath, boolean aFile) { path = csarPath; isFile = aFile; @@ -49,8 +48,9 @@ public class CSAR { csar = null; tempDir = null; tempFile = null; + metaProperties = new LinkedHashMap<>(); } - + @SuppressWarnings("unchecked") public boolean validate() { isValidated = true; @@ -89,8 +89,23 @@ public class CSAR { return false; } - ZipFile zf = null; + _parseAndValidateMetaProperties(); + + if(errorCaught) { + return false; + } + + // validate that external references in the main template actually exist and are accessible + _validateExternalReferences(); + return !errorCaught; + + } + + private void _parseAndValidateMetaProperties() { + + ZipFile zf = null; + try { // validate that it is a valid zip file @@ -99,85 +114,79 @@ public class CSAR { raf.close(); // check if Zip's magic number if (n != 0x504B0304) { - throw new IOException(String.format("\"%s\" is not a valid zip file", csar)); + throw new IOException(String.format("\"%s\" is not a valid zip file", csar)); } - + // validate that it contains the metadata file in the correct location zf = new ZipFile(csar); ZipEntry ze = zf.getEntry("TOSCA-Metadata/TOSCA.meta"); - if(ze == null) { + if (ze == null) { throw new IOException(String.format( "\"%s\" is not a valid CSAR as it does not contain the " + - "required file \"TOSCA.meta\" in the folder \"TOSCA-Metadata\"", csar)); + "required file \"TOSCA.meta\" in the folder \"TOSCA-Metadata\"", csar)); } - + + //Going over expected metadata files and parsing them + for (String metaFile: META_PROPERTIES_FILES) { + + byte ba[] = new byte[4096]; + ze = zf.getEntry(metaFile); + if (ze != null) { + InputStream inputStream = zf.getInputStream(ze); + n = inputStream.read(ba, 0, 4096); + + String md = new String(ba); + md = md.substring(0, (int) n); + Yaml yaml = new Yaml(); + Object mdo = yaml.load(md); + if (!(mdo instanceof LinkedHashMap)) { + throw new IOException(String.format( + "The file \"%s\" in the" + + " CSAR \"%s\" does not contain valid YAML content", ze.getName(), csar)); + } + + String[] split = ze.getName().split("/"); + String fileName = split[split.length - 1]; + + if (!metaProperties.containsKey(fileName)) { + metaProperties.put(fileName, (LinkedHashMap) mdo); + } + } + } + // verify it has "Entry-Definition" - ZipInputStream zipIn = new ZipInputStream(new FileInputStream(csar)); - byte ba[] = new byte[4096]; - while ((ze = zipIn.getNextEntry()) != null) { - if (ze.getName().equals("TOSCA-Metadata/TOSCA.meta")) { - n = zipIn.read(ba,0,4096); - zipIn.close(); - break; - } + String edf = _getMetadata("Entry-Definitions"); + if (edf == null) { + throw new IOException(String.format( + "The CSAR \"%s\" is missing the required metadata " + + "\"Entry-Definitions\" in \"TOSCA-Metadata/TOSCA.meta\"", csar)); } - String md = new String(ba); - md = md.substring(0, (int)n); - Yaml yaml = new Yaml(); - Object mdo = yaml.load(md); - if(!(mdo instanceof LinkedHashMap)) { - throw new IOException(String.format( - "The file \"TOSCA-Metadata/TOSCA.meta\" in the" + - " CSAR \"%s\" does not contain valid YAML content",csar)); - } - - metaData = new Metadata((Map) mdo); - String edf = metaData.getValue("Entry-Definitions"); - if(edf == null) { - throw new IOException(String.format( - "The CSAR \"%s\" is missing the required metadata " + - "\"Entry-Definitions\" in \"TOSCA-Metadata/TOSCA.meta\"",csar)); - } - - //validate that "Entry-Definitions' metadata value points to an existing file in the CSAR - boolean foundEDF = false; - Enumeration entries = zf.entries(); - while (entries.hasMoreElements()) { - ze = entries.nextElement(); - if (ze.getName().equals(edf)) { - foundEDF = true; - break; - } - } - if(!foundEDF) { - throw new IOException(String.format( - "The \"Entry-Definitions\" file defined in the CSAR \"%s\" does not exist",csar)); - } - - } - catch(Exception e) { + //validate that "Entry-Definitions' metadata value points to an existing file in the CSAR + boolean foundEDF = false; + Enumeration entries = zf.entries(); + while (entries.hasMoreElements()) { + ze = entries.nextElement(); + if (ze.getName().equals(edf)) { + foundEDF = true; + break; + } + } + if (!foundEDF) { + throw new IOException(String.format( + "The \"Entry-Definitions\" file defined in the CSAR \"%s\" does not exist", csar)); + } + } catch (Exception e) { ExceptionCollector.appendException("ValidationError: " + e.getMessage()); - errorCaught = true;; + errorCaught = true; } - + try { - if(zf != null) { + if (zf != null) { zf.close(); } + } catch (IOException e) { } - catch(IOException e) { - } - - if(errorCaught) { - return false; - } - - // validate that external references in the main template actually exist and are accessible - _validateExternalReferences(); - - return !errorCaught; - } public void cleanup() { @@ -190,15 +199,12 @@ public class CSAR { } } - public Metadata getMetadata() { - return metaData; - } - private String _getMetadata(String key) { if(!isValidated) { validate(); } - return metaData.getValue(key); + Object value = _getMetaProperty("TOSCA.meta").get(key); + return value != null ? value.toString() : null; } public String getAuthor() { @@ -208,8 +214,16 @@ public class CSAR { public String getVersion() { return _getMetadata("CSAR-Version"); } - - public String getMainTemplate() { + + public LinkedHashMap> getMetaProperties() { + return metaProperties; + } + + private LinkedHashMap _getMetaProperty(String propertiesFile) { + return metaProperties.get(propertiesFile); + } + + public String getMainTemplate() { String entryDef = _getMetadata("Entry-Definitions"); ZipFile zf; boolean ok = false; @@ -259,8 +273,10 @@ public class CSAR { if(desc != null) { return desc; } - metaData.setValue("Description",(String)getMainTemplateYaml().get("description")); - return metaData.getValue("Description"); + + Map metaData = metaProperties.get("TOSCA.meta"); + metaData.put("Description", getMainTemplateYaml().get("description")); + return _getMetadata("Description"); } public String getTempDir() { diff --git a/jtosca/src/test/java/org.openecomp.sdc.toscaparser/JToscaMetadataParse.java b/jtosca/src/test/java/org.openecomp.sdc.toscaparser/JToscaMetadataParse.java new file mode 100644 index 0000000..584a0fd --- /dev/null +++ b/jtosca/src/test/java/org.openecomp.sdc.toscaparser/JToscaMetadataParse.java @@ -0,0 +1,26 @@ +package org.openecomp.sdc.toscaparser; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.File; +import java.util.LinkedHashMap; + +import org.junit.Test; +import org.openecomp.sdc.toscaparser.api.ToscaTemplate; +import org.openecomp.sdc.toscaparser.api.common.JToscaException; + +public class JToscaMetadataParse { + + @Test + public void testMetadataParsedCorrectly() throws JToscaException { + String fileStr = JToscaMetadataParse.class.getClassLoader().getResource("csars/csar_hello_world.csar").getFile(); + File file = new File(fileStr); + ToscaTemplate toscaTemplate = new ToscaTemplate(file.getAbsolutePath(), null, true, null); + LinkedHashMap metadataProperties = toscaTemplate.getMetaProperties("TOSCA.meta"); + assertNotNull(metadataProperties); + Object entryDefinition = metadataProperties.get("Entry-Definitions"); + assertNotNull(entryDefinition); + assertEquals("tosca_helloworld.yaml", entryDefinition); + } +} diff --git a/jtosca/src/test/resources/csars/csar_hello_world.csar b/jtosca/src/test/resources/csars/csar_hello_world.csar new file mode 100644 index 0000000..43ffbbc Binary files /dev/null and b/jtosca/src/test/resources/csars/csar_hello_world.csar differ diff --git a/pom.xml b/pom.xml index f03c0fe..2b3583e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.openecomp.sdc.sdc-distribution-client sdc-main-distribution-client - 1.1.9-SNAPSHOT + 1.1.10-SNAPSHOT pom diff --git a/sdc-distribution-ci/pom.xml b/sdc-distribution-ci/pom.xml index c7b544c..f76457c 100644 --- a/sdc-distribution-ci/pom.xml +++ b/sdc-distribution-ci/pom.xml @@ -6,7 +6,7 @@ org.openecomp.sdc.sdc-distribution-client sdc-main-distribution-client - 1.1.9-SNAPSHOT + 1.1.10-SNAPSHOT sdc-distribution-ci diff --git a/sdc-distribution-client/pom.xml b/sdc-distribution-client/pom.xml index ea13907..d7f0ea7 100644 --- a/sdc-distribution-client/pom.xml +++ b/sdc-distribution-client/pom.xml @@ -6,7 +6,7 @@ org.openecomp.sdc.sdc-distribution-client sdc-main-distribution-client - 1.1.9-SNAPSHOT + 1.1.10-SNAPSHOT sdc-distribution-client diff --git a/sdc-tosca-parser/pom.xml b/sdc-tosca-parser/pom.xml index 6b370ce..f027a96 100644 --- a/sdc-tosca-parser/pom.xml +++ b/sdc-tosca-parser/pom.xml @@ -6,7 +6,7 @@ org.openecomp.sdc.sdc-distribution-client sdc-main-distribution-client - 1.1.9-SNAPSHOT + 1.1.10-SNAPSHOT sdc-tosca-parser @@ -71,7 +71,7 @@ org.openecomp.sdc.sdc-distribution-client jtosca - 0.1.2-SNAPSHOT + 0.1.3-SNAPSHOT @@ -90,10 +90,17 @@ test + + + + + + + - junit - junit - 4.12 + org.testng + testng + 6.11 test @@ -108,7 +115,7 @@ com.google.guava guava 15.0 - test +