From d6f2d611538a5379e340aab26a51a0782555ceec Mon Sep 17 00:00:00 2001 From: AvinashS Date: Thu, 26 Oct 2017 14:27:18 +0530 Subject: Add custom validation To support custom CSAR structure for Amsterdam, add support with schema based validation.This can scale to future enhancements and hardening of validation. Change-Id: I8a20dd2377dafc94346412a8601f1299fa4600c8 Issue-Id: VNFSDK-117 Signed-off-by: AvinashS --- .../org/onap/validation/csar/CsarValidator.java | 137 +++++++++++++++----- .../validation/csar/ValidatorSchemaLoader.java | 142 +++++++++++++++++++++ csarvalidation/src/main/resources/schema/MRF.mf | 1 - .../src/main/resources/schema/TOSCA.meta | 3 +- .../csarvalidationtest/CsarValidatorTest.java | 2 +- csarvalidation/src/test/resources/sample.csar | Bin 5144 -> 0 bytes 6 files changed, 248 insertions(+), 37 deletions(-) create mode 100644 csarvalidation/src/main/java/org/onap/validation/csar/ValidatorSchemaLoader.java delete mode 100644 csarvalidation/src/test/resources/sample.csar diff --git a/csarvalidation/src/main/java/org/onap/validation/csar/CsarValidator.java b/csarvalidation/src/main/java/org/onap/validation/csar/CsarValidator.java index 9ca9c26..9df9e82 100644 --- a/csarvalidation/src/main/java/org/onap/validation/csar/CsarValidator.java +++ b/csarvalidation/src/main/java/org/onap/validation/csar/CsarValidator.java @@ -33,7 +33,7 @@ public class CsarValidator { private static final Logger LOG = LoggerFactory.getLogger(CsarValidator.class); //Schema files - // static private ValidatorSchemaLoader vsl; + static private ValidatorSchemaLoader vsl; // Map of CSAR file and un-zipped file indices static private HashMap csarFiles; @@ -41,6 +41,8 @@ public class CsarValidator { // Map of packageId and CSAR files private static HashMap> csar = new HashMap>(); private static String MAINSERV_TEMPLATE; + private static String MAINSERV_MANIFEST; + /** * * @param packageId @@ -67,6 +69,13 @@ public class CsarValidator { //deleteDirectory(); LOG.error("CSAR %s is not a valid CSAR/ZIP file! ", e1); } + + + try { + vsl = new ValidatorSchemaLoader(); + } catch (Exception e) { + e.printStackTrace(); + } } /** * @@ -76,7 +85,8 @@ public class CsarValidator { boolean vsm = validateCsarMeta(); - boolean vtm = validateToscaMeta(); + // boolean vtm = validateToscaMeta__(); + boolean vtm = validateAndScanToscaMeta(); boolean vms = validateMainService(); @@ -84,7 +94,7 @@ public class CsarValidator { return true; } - //In future return the status handler object instead. + return false; } @@ -111,6 +121,7 @@ public class CsarValidator { return false; } } + /** * * @return true if csar meta data validation is successful @@ -165,44 +176,67 @@ public class CsarValidator { * * @return true csar tosca meta validation is successful */ - public static boolean validateToscaMeta() { + public static boolean validateAndScanToscaMeta() { String cfile = csarFiles.get(CommonConstants.TOSCA_META); - if(StringUtils.isEmpty(cfile)) { + + if (!validateToscaMeta(cfile)) { return false; } + try { if (cfile.contains(System.getProperty("file.separator") + CommonConstants.TOSCA_METADATA + System.getProperty("file.separator") + CommonConstants.TOSCA_META)) { - String value = CheckEntryFor(cfile, "Entry-Definitions:"); - String[] splitPath = value.split("/"); - String subValue = splitPath[splitPath.length - 1]; - - if (value.isEmpty() || subValue.isEmpty()) { - return false; - - //Check if Entry-Defintions pointed file exists in CSAR - } else { - if (!(null == csarFiles.get(value))) { - MAINSERV_TEMPLATE = csarFiles.get(value); - return true; - } - else if (!(null == csarFiles.get(subValue))) { - MAINSERV_TEMPLATE = csarFiles.get(subValue); - return true; - } - else { - MAINSERV_TEMPLATE = CommonConstants.MAINSERV_TEMPLATE; - } + MAINSERV_MANIFEST = checkAndGetMRF(cfile,"Entry-Manifest"); + if (MAINSERV_MANIFEST == null) { + MAINSERV_MANIFEST = CommonConstants.MAINSERV_MANIFEST; } + + MAINSERV_TEMPLATE = checkAndGetMRF(cfile,"Entry-Definitions"); + if (MAINSERV_TEMPLATE == null) { + MAINSERV_TEMPLATE = CommonConstants.MAINSERV_TEMPLATE; + } + + return true; } - } catch (IOException | NullPointerException e) { - LOG.error("CSAR_TOSCA_VALIDATION" + ":" + "Could not read file %s ! " +ErrorCodes.FILE_IO+ " " +ErrorCodes.RESOURCE_MISSING, e); - throw new ValidationException(ErrorCodes.RESOURCE_MISSING); + } catch (Exception e) { + LOG.error("Parsing error"); } + return false; } + + private static String checkAndGetMRF(String mrfFile, String attribute) { + try { + String value = CheckEntryFor(mrfFile, attribute); + String mrfCsarEntry = null; + + //Rel-1 & SOL004 Entry-Definitions is optional + if (! StringUtils.isEmpty(value)) { + + if(value.contains("Definitions/")) + { + String[] splitPath = value.split("/"); + mrfCsarEntry = csarFiles.get(splitPath[splitPath.length - 1]); + // csarEntry = csarFiles.get(subValue); + } + else { //Hack to support non-compliant "Entry-Definitions:" format + mrfCsarEntry = csarFiles.get(value); + } + + if (null != mrfCsarEntry) { + return mrfCsarEntry; + } + } + } catch (IOException | NullPointerException e) { + LOG.error("CSAR_TOSCA_VALIDATION" + ":" + "Could not read file %s ! " +ErrorCodes.FILE_IO+ " " +ErrorCodes.RESOURCE_MISSING); + throw new ValidationException(ErrorCodes.RESOURCE_MISSING); + } + + return null; + } + /** * * @return true csar validation is successful @@ -217,11 +251,22 @@ public class CsarValidator { @SuppressWarnings("unused") boolean mfResult = CheckEntryFor(CommonConstants.MAINSERV_MANIFEST, mListMetadata, key); - String mrfFile = MAINSERV_TEMPLATE; - if(!Paths.get(mrfFile).isAbsolute()){ - mrfFile = csarFiles.get(FilenameUtils.getName(mrfFile)); + + String mainServManifest = MAINSERV_MANIFEST; + if(!Paths.get(mainServManifest).isAbsolute()){ + mainServManifest = csarFiles.get(FilenameUtils.getName(mainServManifest)); } - if(StringUtils.isEmpty(mrfFile)){ + // Rel-2 SOL004 requirement + if(StringUtils.isEmpty(mainServManifest)){ + //Do nothing for Rel-1 + //return false; + } + + String mainservTemplate = MAINSERV_TEMPLATE; + if(!Paths.get(MAINSERV_TEMPLATE).isAbsolute()){ + mainservTemplate = csarFiles.get(FilenameUtils.getName(mainservTemplate)); + } + if(StringUtils.isEmpty(mainservTemplate)){ return false; } return true; @@ -233,7 +278,8 @@ public class CsarValidator { for (String strLine : lines) { if (!attribute.isEmpty() && strLine.contains(attribute)) { - return strLine.substring(attribute.length(), strLine.length()).trim(); + String entry = strLine.substring(attribute.length(), strLine.length()).trim(); + return entry.replaceFirst(":","").trim(); } } return null; @@ -293,6 +339,31 @@ public class CsarValidator { return true; } + /** + * + * @return true if csar meta data validation is successful + */ + private static boolean validateToscaMeta(String cfile) { + + if (StringUtils.isEmpty(cfile)) { + return false; + } + else { + File file = new File(cfile); + + Yaml yaml = new Yaml(); + + + Map toscaMeta = null; + try { + toscaMeta = (Map) yaml.load(new FileInputStream(file)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return toscaMeta.keySet().containsAll((vsl.getToscaMeta().keySet())); + } + } + public static HashMap> getCsar() { return csar; } diff --git a/csarvalidation/src/main/java/org/onap/validation/csar/ValidatorSchemaLoader.java b/csarvalidation/src/main/java/org/onap/validation/csar/ValidatorSchemaLoader.java new file mode 100644 index 0000000..aa68cfb --- /dev/null +++ b/csarvalidation/src/main/java/org/onap/validation/csar/ValidatorSchemaLoader.java @@ -0,0 +1,142 @@ +/** + * Copyright 2017 Huawei Technologies Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.onap.validation.csar; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Stream; + +import org.apache.commons.io.FilenameUtils; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.scanner.ScannerException; + +public class ValidatorSchemaLoader { + + private static final Logger LOG = LoggerFactory.getLogger(ValidatorSchemaLoader.class); + + // Map of Schema files + private static Map toscaMeta; + private static Map csarentryd; + private static Map mrfYaml; + private static Map mrfManifest; + + // List of configured schemas + static List schemaFileList = new ArrayList(); + + // SOL004 rule files + static HashMap optionOneSchema; + static HashMap optionTwoSchema; + + public ValidatorSchemaLoader() throws Exception { + + + try { + loadResources(); + } catch ( FileNotFoundException e1) { + LOG.error("Schema file not found or schema repository corrupted"); + + } + } + + + private boolean loadResources() throws FileNotFoundException { + + ClassLoader classLoader = getClass().getClassLoader(); + + final InputStream is = classLoader.getResourceAsStream("./schema/"); + final InputStreamReader isr = new InputStreamReader(is, StandardCharsets.UTF_8); + final BufferedReader br = new BufferedReader(isr); + + + br.lines().filter(Objects::nonNull) + .forEach((String e) -> { + + + File file = new File(getClass().getClassLoader().getResource("schema/"+e.toString()).getFile()); + if (!file.isDirectory() && ( + FilenameUtils.isExtension(file.getName(), "yaml") || + FilenameUtils.isExtension(file.getName(), "mf") || + FilenameUtils.isExtension(file.getName(), "meta"))) { + + Yaml yaml = new Yaml(); + + switch (file.getName()) { + case "TOSCA.meta" : + try { + toscaMeta = (Map) yaml.load(new FileInputStream(file)); + } catch (ScannerException | FileNotFoundException e1) { + LOG.error("Schema files %s format is not as per standard prescribed",file.getName()); + } + break; + case "CSAR.meta" : + try { + csarentryd = (Map) yaml.load(new FileInputStream(file)); + } catch (ScannerException | FileNotFoundException e2) { + LOG.error("Schema files %s format is not as per standard prescribed",file.getName()); + } + break; + case "MRF.yaml" : + try { + mrfYaml = (Map) yaml.load(new FileInputStream(file)); + } catch (ScannerException | FileNotFoundException e2) { + LOG.error("Schema files %s format is not as per standard prescribed",file.getName()); + } + break; + case "MRF.mf" : + try { + mrfManifest = (Map) yaml.load(new FileInputStream(file)); + } catch (ScannerException | FileNotFoundException e2) { + LOG.error("Schema files %s format is not as per standard prescribed",file.getName()); + } + break; + } + } + schemaFileList.add(e); + }); + + return true; + } + + public static Map getToscaMeta() { + return toscaMeta; + } + + public static Map getCsarentryd() { + return csarentryd; + } + + public static Map getMrfYaml() { + return mrfYaml; + } + + public static Map getMrfManifest() { + return mrfManifest; + } + + public static List getSchemaFileList() { + return schemaFileList; + } +} \ No newline at end of file diff --git a/csarvalidation/src/main/resources/schema/MRF.mf b/csarvalidation/src/main/resources/schema/MRF.mf index 4b7d65d..70a2aff 100644 --- a/csarvalidation/src/main/resources/schema/MRF.mf +++ b/csarvalidation/src/main/resources/schema/MRF.mf @@ -3,4 +3,3 @@ metadata: vnf_provider_id: UTF-8 vnf_package_version: Double vnf_release_data_time: String_IETF_RFC-3339 - { get_input: myinput } diff --git a/csarvalidation/src/main/resources/schema/TOSCA.meta b/csarvalidation/src/main/resources/schema/TOSCA.meta index 837b45a..4b8de65 100644 --- a/csarvalidation/src/main/resources/schema/TOSCA.meta +++ b/csarvalidation/src/main/resources/schema/TOSCA.meta @@ -1,4 +1,3 @@ -TOSCA-Meta-File-Version: Double +TOSCA-Meta-Version: Double CSAR-Version: Double -Created-By: String Entry-Definitions: { get_input: myinput } \ No newline at end of file diff --git a/csarvalidation/src/test/java/org/onap/validation/csarvalidationtest/CsarValidatorTest.java b/csarvalidation/src/test/java/org/onap/validation/csarvalidationtest/CsarValidatorTest.java index 826decb..cc1ada7 100644 --- a/csarvalidation/src/test/java/org/onap/validation/csarvalidationtest/CsarValidatorTest.java +++ b/csarvalidation/src/test/java/org/onap/validation/csarvalidationtest/CsarValidatorTest.java @@ -176,7 +176,7 @@ public class CsarValidatorTest { private void testValidateToscaMeta(CsarValidator cv) { - boolean result = CsarValidator.validateToscaMeta(); + boolean result = CsarValidator.validateAndScanToscaMeta(); assertEquals(true, result == true); } diff --git a/csarvalidation/src/test/resources/sample.csar b/csarvalidation/src/test/resources/sample.csar deleted file mode 100644 index 2bcb631..0000000 Binary files a/csarvalidation/src/test/resources/sample.csar and /dev/null differ -- cgit 1.2.3-korg