diff options
Diffstat (limited to 'openecomp-be/lib')
10 files changed, 420 insertions, 1 deletions
diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java index e70ad923a2..f510febbcf 100644 --- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java +++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/main/java/org/openecomp/core/utilities/file/FileUtils.java @@ -1,5 +1,5 @@ /* - * Copyright © 2016-2017 European Support Limited + * Copyright © 2016-2018 European Support Limited * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,13 +22,20 @@ import org.openecomp.core.utilities.json.JsonUtil; import org.openecomp.sdc.tosca.services.YamlUtil; import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collections; import java.util.Enumeration; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.zip.ZipEntry; @@ -277,4 +284,44 @@ public class FileUtils { } + /** + * Write files and folders map to disk in the given path + * + * @param fileContentHandler the file content handler + * @param dir the dir + * @return a map containing file names and their absolute paths + * @throws IOException the io exception + */ + public static Map<String, String> writeFilesFromFileContentHandler(FileContentHandler + fileContentHandler, + Path dir) + throws IOException { + + File file; + File dirFile = dir.toFile(); + Map<String, String> filePaths = new HashMap<>(); + for (Map.Entry<String, byte[]> fileEntry : fileContentHandler.getFiles().entrySet()) { + file = new File(dirFile, fileEntry.getKey()); + file.getParentFile().mkdirs(); + filePaths.put(fileEntry.getKey(), file.getAbsolutePath()); + try (FileOutputStream fop = new FileOutputStream(file.getAbsolutePath());) { + fop.write(fileEntry.getValue()); + fop.flush(); + } + } + + return filePaths; + } + + /** + * Verify whether the provided extension is valid Yaml/Yml extension or not. + * + * @param fileExtension the file extension + * @return the boolean + */ + public static boolean isValidYamlExtension(String fileExtension){ + return fileExtension.equalsIgnoreCase(FileExtension.YML.getDisplayName()) || + fileExtension.equalsIgnoreCase(FileExtension.YAML.getDisplayName()); + } + } diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java index e32aa39904..3c00d40a6b 100644 --- a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java +++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/java/org/openecomp/core/utilities/file/FileUtilsTest.java @@ -1,12 +1,36 @@ +/* + * Copyright © 2016-2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.openecomp.core.utilities.file; +import org.apache.commons.io.IOUtils; +import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum; import org.testng.annotations.Test; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; import java.util.function.Function; import static org.testng.Assert.assertTrue; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertEquals; /** * @author EVITALIY @@ -45,4 +69,35 @@ public class FileUtilsTest { public void testReadViaInputStreamNotFound() throws Exception { FileUtils.readViaInputStream("notfound.txt", TEST_FUNCTION); } + + @Test + public void testWriteFilesFromFileContentHandler() throws IOException { + Path dir = Files.createTempDirectory("CSAR_" + System.currentTimeMillis()); + try { + byte[] uploadedFileData = IOUtils.toByteArray( + FileUtilsTest.class.getResource("resource-Spgw-csar-ZTE" + + ".csar")); + FileContentHandler contentMap = FileUtils.getFileContentMapFromZip(uploadedFileData); + Map<String, String> filePaths = FileUtils.writeFilesFromFileContentHandler(contentMap, + dir); + + assertFalse(filePaths.isEmpty()); + assertEquals(filePaths.size(), 18); + for (Map.Entry<String, String> fileEntry : filePaths.entrySet()) { + File f = new File(fileEntry.getValue()); + assertTrue(f.exists()); + } + } + finally { + org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile()); + } + } + + @Test + public void testIsValidYamlExtension() throws IOException { + assertTrue(FileUtils.isValidYamlExtension("yaml")); + assertTrue(FileUtils.isValidYamlExtension("yml")); + assertFalse(FileUtils.isValidYamlExtension("yml1")); + assertFalse(FileUtils.isValidYamlExtension("zip")); + } }
\ No newline at end of file diff --git a/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/resources/org/openecomp/core/utilities/file/resource-Spgw-csar-ZTE.csar b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/resources/org/openecomp/core/utilities/file/resource-Spgw-csar-ZTE.csar Binary files differnew file mode 100644 index 0000000000..58c3ddd074 --- /dev/null +++ b/openecomp-be/lib/openecomp-core-lib/openecomp-utilities-lib/src/test/resources/org/openecomp/core/utilities/file/resource-Spgw-csar-ZTE.csar diff --git a/openecomp-be/lib/openecomp-tosca-lib/pom.xml b/openecomp-be/lib/openecomp-tosca-lib/pom.xml index 14b4d7734c..8880c2bd07 100644 --- a/openecomp-be/lib/openecomp-tosca-lib/pom.xml +++ b/openecomp-be/lib/openecomp-tosca-lib/pom.xml @@ -27,6 +27,11 @@ </dependency> <dependency> <groupId>org.openecomp.sdc</groupId> + <artifactId>openecomp-sdc-validation-api</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.openecomp.sdc</groupId> <artifactId>openecomp-sdc-datatypes-lib</artifactId> <version>${project.version}</version> </dependency> @@ -69,6 +74,11 @@ <artifactId>openecomp-configuration-management-api</artifactId> <version>${openecomp.sdc.common.version}</version> </dependency> + <dependency> + <groupId>org.openecomp.sdc.sdc-tosca</groupId> + <artifactId>sdc-tosca</artifactId> + <version>${sdc-tosca-parser.version}</version> + </dependency> </dependencies> <build> <plugins> diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java new file mode 100644 index 0000000000..59d7acccd4 --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/ToscaValidationService.java @@ -0,0 +1,31 @@ +/* + * Copyright © 2016-2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openecomp.sdc.tosca.services; + +import org.openecomp.core.utilities.file.FileContentHandler; +import org.openecomp.sdc.datatypes.error.ErrorMessage; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public interface ToscaValidationService { + + public Map<String, List<ErrorMessage>> validate(FileContentHandler fileContentHandler) + throws IOException; + +} diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java new file mode 100644 index 0000000000..1c2c408c1a --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImpl.java @@ -0,0 +1,159 @@ +/* + * Copyright © 2016-2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openecomp.sdc.tosca.services.impl; + +import org.apache.commons.io.FilenameUtils; +import org.openecomp.core.utilities.file.FileContentHandler; +import org.openecomp.core.utilities.file.FileUtils; +import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum; +import org.openecomp.core.validation.ErrorMessageCode; +import org.openecomp.core.validation.errors.ErrorMessagesFormatBuilder; +import org.openecomp.sdc.datatypes.error.ErrorLevel; +import org.openecomp.sdc.datatypes.error.ErrorMessage; +import org.openecomp.sdc.logging.api.Logger; +import org.openecomp.sdc.logging.api.LoggerFactory; +import org.openecomp.sdc.tosca.parser.config.ConfigurationManager; +import org.openecomp.sdc.tosca.parser.exceptions.SdcToscaParserException; +import org.openecomp.sdc.tosca.parser.impl.SdcToscaParserFactory; +import org.openecomp.sdc.tosca.services.ToscaValidationService; +import org.openecomp.sdc.toscaparser.api.common.JToscaValidationIssue; +import org.yaml.snakeyaml.Yaml; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class ToscaValidationServiceImpl implements ToscaValidationService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ToscaValidationServiceImpl.class); + private static final String SDCPARSER_JTOSCA_VALIDATIONISSUE_CONFIG = + "SDCParser_jtosca-validation-issue-configuration.yaml"; + private static final String SDCPARSER_ERROR_CONFIG = "SDCParser_error-configuration.yaml"; + private static final String TOSCA_DEFINITION_VERSION = "tosca_definitions_version"; + + static { + // Override default SDC Parser configuration + ConfigurationManager configurationManager = ConfigurationManager.getInstance(); + configurationManager.setJtoscaValidationIssueConfiguration(SDCPARSER_JTOSCA_VALIDATIONISSUE_CONFIG); + configurationManager.setErrorConfiguration(SDCPARSER_ERROR_CONFIG); + SdcToscaParserFactory.setConfigurationManager(configurationManager); + } + + @Override + public Map<String, List<ErrorMessage>> validate(FileContentHandler fileContentHandler) + throws IOException { + + Path dir = + Files.createTempDirectory(OnboardingTypesEnum.CSAR + "_" + System.currentTimeMillis()); + try { + // Write temporary files and folders to File System + Map<String, String> filePaths = FileUtils.writeFilesFromFileContentHandler + (fileContentHandler, dir); + // Process Tosca Yaml validation + return processToscaYamls(filePaths); + } finally { + // Cleanup temporary files and folders from file system + org.apache.commons.io.FileUtils.deleteDirectory(dir.toFile()); + } + } + + private Map<String, List<ErrorMessage>> processToscaYamls(Map<String, String> filePaths) { + Map<String, String> validFilePaths = getValidFilePaths(filePaths); + Map<String, List<ErrorMessage>> validationIssues = new HashMap<>(); + + // Process Yaml Files + for (Map.Entry<String, String> fileEntry : validFilePaths.entrySet()) { + try { + SdcToscaParserFactory factory = SdcToscaParserFactory.getInstance(); + factory.getSdcCsarHelper(fileEntry.getValue()); + processValidationIssues(fileEntry.getKey(), factory, validationIssues); + } catch (SdcToscaParserException stpe) { + LOGGER.error("SDC Parser Exception from SDC Parser Library : " + stpe); + ErrorMessage.ErrorMessageUtil.addMessage(fileEntry.getKey(), validationIssues).add( + new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder + .getErrorWithParameters(new ErrorMessageCode("JE000"), "Unexpected Error " + + "occurred"))); + } + catch (RuntimeException rte) { + LOGGER.error("Runtime Exception from SDC Parser Library : " + rte); + ErrorMessage.ErrorMessageUtil.addMessage(fileEntry.getKey(), validationIssues).add( + new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder + .getErrorWithParameters(new ErrorMessageCode("JE000"), "Unexpected Error " + + "occurred"))); + } + } + return validationIssues; + } + + private Map<String, String> getValidFilePaths(Map<String, String> filePaths) { + return filePaths.entrySet() + .stream() + .filter(map -> FileUtils.isValidYamlExtension(FilenameUtils.getExtension(map.getKey())) + && isToscaYaml(map.getValue())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private boolean isToscaYaml(String filePath) { + boolean retValue = false; + + try (InputStream input = new BufferedInputStream(new FileInputStream(new File(filePath)));) { + Yaml yaml = new Yaml(); + LinkedHashMap<String,Object> data = (LinkedHashMap) yaml.load(input); + if(data.get(TOSCA_DEFINITION_VERSION) != null) { + retValue = true; + } + } + catch(Exception e){ + LOGGER.info("Ignore the exception as the input file may not be a Tosca Yaml; let the " + + "default value return", e); + } + return retValue; + } + + private void processValidationIssues(String fileName, SdcToscaParserFactory factory, Map<String, + List<ErrorMessage>> validationIssues) { + + List<JToscaValidationIssue> criticalsReport = factory.getCriticalExceptions(); + criticalsReport.stream().forEach(err -> + ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add( + new ErrorMessage(ErrorLevel.ERROR, ErrorMessagesFormatBuilder + .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage())))); + + List<JToscaValidationIssue> warningsReport = factory.getWarningExceptions(); + warningsReport.stream().forEach(err -> + ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add( + new ErrorMessage(ErrorLevel.WARNING, ErrorMessagesFormatBuilder + .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage())))); + + List<JToscaValidationIssue> notAnalyzedReport = factory.getNotAnalyzadExceptions(); + notAnalyzedReport.stream().forEach(err -> + ErrorMessage.ErrorMessageUtil.addMessage(fileName, validationIssues).add( + new ErrorMessage(ErrorLevel.WARNING, ErrorMessagesFormatBuilder + .getErrorWithParameters(new ErrorMessageCode(err.getCode()), err.getMessage())))); + + } + +} diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml new file mode 100644 index 0000000000..f5c20aa347 --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_error-configuration.yaml @@ -0,0 +1,22 @@ +# Errors +errors: + FILE_NOT_FOUND: { + code: TP0001, + failOnError: true, + message: "Error: CSAR file not found." + } + BAD_FORMAT: { + code: TP0002, + failOnError: true, + message: "Error: CSAR file bad format. Check the log for details." + } + CONFORMANCE_LEVEL_ERROR: { + code: TP0003, + failOnError: false, + message: "Error: CSAR version is unsupported. Parser supports versions %s to %s." + } + GENERAL_ERROR: { + code: TP0004, + failOnError: true, + message: "Error: an unexpected internal error occured." + }
\ No newline at end of file diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml new file mode 100644 index 0000000000..f4bb949c80 --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/main/resources/config/SDCParser_jtosca-validation-issue-configuration.yaml @@ -0,0 +1,49 @@ +# jTosca validation issues +#by error code, type the validation issue to be CRITICAL/WARNING +# since Conformance level considered to this type. for example: +#JE001: +# - issueType: WARNING +# sinceCsarConformanceLevel: 3.0 +# - issueType: WARNING +# sinceCsarConformanceLevel: 5.0 +validationIssues: + # TypeMismatchError + JE001: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + # MissingType + JE002: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #MissingRequiredFieldError + JE003: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #MissingRequiredFieldError2 + JE004: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #InvalidGroupTargetException + JE005: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #Schema definition of \"%s\" has \"status\" attribute with an invalid value + JE006: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #The unit \"%s\" is not valid + JE007: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #ValidationError + JE008: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #ValueError: Expected max 2 arguments for function \"get_input\" but received \"%s\"",args.size()) + JE009: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0 + #MissingRequiredFieldError3 + JE010: + - issueType: WARNING + sinceCsarConformanceLevel: 3.0
\ No newline at end of file diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java b/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java new file mode 100644 index 0000000000..c9ceb363a4 --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/test/java/org/openecomp/sdc/tosca/services/impl/ToscaValidationServiceImplTest.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2016-2018 European Support Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openecomp.sdc.tosca.services.impl; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import org.openecomp.core.utilities.file.FileContentHandler; +import org.openecomp.core.utilities.orchestration.OnboardingTypesEnum; +import org.openecomp.sdc.common.utils.CommonUtil; +import org.openecomp.sdc.datatypes.error.ErrorMessage; +import org.openecomp.sdc.tosca.services.ToscaValidationService; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertFalse; + +public class ToscaValidationServiceImplTest { + + @Test + public void validateCSARContentErrorHandling() throws IOException { + String resName = "/mock/validationService/csar/resource-Spgw-csar-ZTE.csar"; + byte[] uploadedFileData = IOUtils.toByteArray(this.getClass().getResource(resName)); + FileContentHandler contentMap = + CommonUtil.validateAndUploadFileContent(OnboardingTypesEnum.CSAR, uploadedFileData); + ToscaValidationService handler = new ToscaValidationServiceImpl(); + Map<String, List<ErrorMessage>> errors = handler.validate(contentMap); + assertFalse(errors.isEmpty()); + } + +} diff --git a/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar b/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar Binary files differnew file mode 100644 index 0000000000..58c3ddd074 --- /dev/null +++ b/openecomp-be/lib/openecomp-tosca-lib/src/test/resources/mock/validationService/csar/resource-Spgw-csar-ZTE.csar |