SDC-TOSCA Parser Integration

SDC Onboarding Tosca Library to call SDC-Tosca Parser

Change-Id: I12c92def4ff8e17431d73b0d39f62d5d79ea6f21
Issue-ID: SDC-1043
Signed-off-by: priyanshu <pagarwal@amdocs.com>

7 files changed, 317 insertions, 0 deletions

diff --git a/openecomp-be/lib/openecomp-tosca-lib/pom.xml b/openecomp-be/lib/openecomp-tosca-lib/pom.xml
index 14b4d7734c..8880c2bd07 100644
--- a/openecomp-be/lib/openecomp-tosca-lib/pom.xml
+++ b/openecomp-be/lib/openecomp-tosca-lib/pom.xml
@@ -27,6 +27,11 @@
         </dependency>
         <dependency>
             <groupId>org.openecomp.sdc</groupId>
+            <artifactId>openecomp-sdc-validation-api</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.openecomp.sdc</groupId>
             <artifactId>openecomp-sdc-datatypes-lib</artifactId>
             <version>${project.version}</version>
         </dependency>
@@ -69,6 +74,11 @@
             <artifactId>openecomp-configuration-management-api</artifactId>
             <version>${openecomp.sdc.common.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.openecomp.sdc.sdc-tosca</groupId>
+            <artifactId>sdc-tosca</artifactId>
+            <version>${sdc-tosca-parser.version}</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java
new file mode 100644
index 0000000000..59d7acccd4
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright © 2016-2018 European Support Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.openecomp.sdc.tosca.services;
+
+import org.openecomp.core.utilities.file.FileContentHandler;
+import org.openecomp.sdc.datatypes.error.ErrorMessage;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public interface ToscaValidationService {
+
+  public Map<String, List<ErrorMessage>> validate(FileContentHandler fileContentHandler)
+      throws IOException;
+
+}
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java
new file mode 100644
index 0000000000..1c2c408c1a
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright © 2016-2018 European Support Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.openecomp.sdc.tosca.services.impl;
+
+import org.apache.commons.io.FilenameUtils;
+import org.openecomp.core.utilities.file.FileContentHandler;
+import org.openecomp.core.utilities.file.FileUtils;
+import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
+import org.openecomp.core.validation.ErrorMessageCode;
+import org.openecomp.core.validation.errors.ErrorMessagesFormatBuilder;
+import org.openecomp.sdc.datatypes.error.ErrorLevel;
+import org.openecomp.sdc.datatypes.error.ErrorMessage;
+import org.openecomp.sdc.logging.api.Logger;
+import org.openecomp.sdc.logging.api.LoggerFactory;
+import org.openecomp.sdc.tosca.parser.config.ConfigurationManager;
+import org.openecomp.sdc.tosca.parser.exceptions.SdcToscaParserException;
+import org.openecomp.sdc.tosca.parser.impl.SdcToscaParserFactory;
+import org.openecomp.sdc.tosca.services.ToscaValidationService;
+import org.openecomp.sdc.toscaparser.api.common.JToscaValidationIssue;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ToscaValidationServiceImpl implements ToscaValidationService {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(ToscaValidationServiceImpl.class);
+  private static final String SDCPARSER_JTOSCA_VALIDATIONISSUE_CONFIG =
+      "SDCParser_jtosca-validation-issue-configuration.yaml";
+  private static final String SDCPARSER_ERROR_CONFIG = "SDCParser_error-configuration.yaml";
+  private static final String TOSCA_DEFINITION_VERSION = "tosca_definitions_version";
+
+  static {
+    // Override default SDC Parser configuration
+    ConfigurationManager configurationManager = ConfigurationManager.getInstance();
+    configurationManager.setJtoscaValidationIssueConfiguration(SDCPARSER_JTOSCA_VALIDATIONISSUE_CONFIG);
+    configurationManager.setErrorConfiguration(SDCPARSER_ERROR_CONFIG);
+    SdcToscaParserFactory.setConfigurationManager(configurationManager);
+  }
+
+  @Override
+  public Map<String, List<ErrorMessage>> validate(FileContentHandler fileContentHandler)
+      throws IOException {
+
+    Path dir =
+        Files.createTempDirectory(OnboardingTypesEnum.CSAR + "_" + System.currentTimeMillis());
+    try {
+      // Write temporary files and folders to File System
+      Map<String, String> filePaths = FileUtils.writeFilesFromFileContentHandler
+          (fileContentHandler, dir);
+      // Process Tosca Yaml validation
+      return processToscaYamls(filePaths);
+    } finally {
+      // Cleanup temporary files and folders from file system
+      org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile());
+    }
+  }
+
+  private Map<String, List<ErrorMessage>> processToscaYamls(Map<String, String> filePaths) {
+    Map<String, String> validFilePaths = getValidFilePaths(filePaths);
+    Map<String, List<ErrorMessage>> validationIssues = new HashMap<>();
+
+    // Process Yaml Files
+    for (Map.Entry<String, String> fileEntry : validFilePaths.entrySet()) {
+      try {
+        SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance();
+        factory.getSdcCsarHelper(fileEntry.getValue());
+        processValidationIssues(fileEntry.getKey(), factory, validationIssues);
+      } catch (SdcToscaParserException stpe) {
+        LOGGER.error("SDC Parser Exception from SDC Parser Library : " + stpe);
+        ErrorMessage.ErrorMessageUtil.addMessage(fileEntry.getKey(), validationIssues).add(
+            new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder
+                .getErrorWithParameters(new ErrorMessageCode("JE000"), "Unexpected Error "
+                    + "occurred")));
+      }
+      catch (RuntimeException rte) {
+        LOGGER.error("Runtime Exception from SDC Parser Library : " + rte);
+        ErrorMessage.ErrorMessageUtil.addMessage(fileEntry.getKey(), validationIssues).add(
+            new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder
+                .getErrorWithParameters(new ErrorMessageCode("JE000"), "Unexpected Error "
+                    + "occurred")));
+      }
+    }
+    return validationIssues;
+  }
+
+  private Map<String, String> getValidFilePaths(Map<String, String> filePaths) {
+    return filePaths.entrySet()
+        .stream()
+        .filter(map -> FileUtils.isValidYamlExtension(FilenameUtils.getExtension(map.getKey()))
+            && isToscaYaml(map.getValue()))
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+  }
+
+  private boolean isToscaYaml(String filePath) {
+   boolean retValue = false;
+
+    try (InputStream input = new BufferedInputStream(new FileInputStream(new File(filePath)));) {
+      Yaml yaml = new Yaml();
+      LinkedHashMap<String,Object> data = (LinkedHashMap) yaml.load(input);
+      if(data.get(TOSCA_DEFINITION_VERSION) != null) {
+        retValue = true;
+      }
+    }
+    catch(Exception e){
+      LOGGER.info("Ignore the exception as the input file may not be a Tosca Yaml; let the " +
+          "default value return", e);
+    }
+    return retValue;
+  }
+
+  private void processValidationIssues(String fileName, SdcToscaParserFactory factory, Map<String,
+      List<ErrorMessage>> validationIssues) {
+
+    List<JToscaValidationIssue> criticalsReport = factory.getCriticalExceptions();
+    criticalsReport.stream().forEach(err ->
+        ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add(
+            new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder
+                .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage()))));
+
+    List<JToscaValidationIssue> warningsReport = factory.getWarningExceptions();
+    warningsReport.stream().forEach(err ->
+        ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add(
+            new ErrorMessage(ErrorLevel.WARNING, ErrorMessagesFormatBuilder
+                .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage()))));
+
+    List<JToscaValidationIssue> notAnalyzedReport = factory.getNotAnalyzadExceptions();
+    notAnalyzedReport.stream().forEach(err ->
+        ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add(
+            new ErrorMessage(ErrorLevel.WARNING, ErrorMessagesFormatBuilder
+                .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage()))));
+
+  }
+
+}
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml
new file mode 100644
index 0000000000..f5c20aa347
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml
@@ -0,0 +1,22 @@
+# Errors
+errors:
+    FILE_NOT_FOUND: {
+        code: TP0001,
+        failOnError: true,
+        message: "Error: CSAR file not found."
+    }
+    BAD_FORMAT: {
+        code: TP0002,
+        failOnError: true,
+        message: "Error: CSAR file bad format. Check the log for details."
+    }
+    CONFORMANCE_LEVEL_ERROR: {
+        code: TP0003,
+        failOnError: false,
+        message: "Error: CSAR version is unsupported. Parser supports versions %s to %s." 
+    }
+    GENERAL_ERROR: {
+        code: TP0004,
+        failOnError: true,
+        message: "Error: an unexpected internal error occured."
+    }
\ No newline at end of file
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml
new file mode 100644
index 0000000000..f4bb949c80
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml
@@ -0,0 +1,49 @@
+# jTosca validation issues 
+#by error code, type the validation issue to be CRITICAL/WARNING 
+# since Conformance level considered to this type. for example:
+#JE001: 
+#      - issueType: WARNING
+#        sinceCsarConformanceLevel: 3.0     
+#      - issueType: WARNING
+#        sinceCsarConformanceLevel: 5.0
+validationIssues:
+    # TypeMismatchError
+    JE001: 
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    # MissingType
+    JE002:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #MissingRequiredFieldError
+    JE003:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #MissingRequiredFieldError2
+    JE004:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #InvalidGroupTargetException
+    JE005:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #Schema definition of \"%s\" has \"status\" attribute with an invalid value
+    JE006:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #The unit \"%s\" is not valid
+    JE007:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #ValidationError
+    JE008:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #ValueError: Expected max 2 arguments for function \"get_input\" but received \"%s\"",args.size())
+    JE009:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
+    #MissingRequiredFieldError3
+    JE010:
+        - issueType: WARNING
+          sinceCsarConformanceLevel: 3.0
\ No newline at end of file
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java b/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java
new file mode 100644
index 0000000000..c9ceb363a4
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright © 2016-2018 European Support Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.openecomp.sdc.tosca.services.impl;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+import org.openecomp.core.utilities.file.FileContentHandler;
+import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum;
+import org.openecomp.sdc.common.utils.CommonUtil;
+import org.openecomp.sdc.datatypes.error.ErrorMessage;
+import org.openecomp.sdc.tosca.services.ToscaValidationService;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+
+public class ToscaValidationServiceImplTest {
+
+  @Test
+  public void validateCSARContentErrorHandling() throws IOException {
+    String resName = "/mock/validationService/csar/resource-Spgw-csar-ZTE.csar";
+    byte[] uploadedFileData = IOUtils.toByteArray(this.getClass().getResource(resName));
+    FileContentHandler contentMap =
+        CommonUtil.validateAndUploadFileContent(OnboardingTypesEnum.CSAR, uploadedFileData);
+    ToscaValidationService handler = new ToscaValidationServiceImpl();
+    Map<String, List<ErrorMessage>> errors = handler.validate(contentMap);
+    assertFalse(errors.isEmpty());
+  }
+
+}
diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar b/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar
new file mode 100644
index 0000000000..58c3ddd074
--- /dev/null
+++ b/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar