From 7e6fe8c29c5a5cfa5caf6ab47b30280e1fc20432 Mon Sep 17 00:00:00 2001 From: "mark.j.leonard" Date: Thu, 7 Jun 2018 16:45:38 +0100 Subject: Add support for loading VNF Catalog XML files Issue-ID: AAI-1214 Change-Id: I5d0eb3456916e6f3e5ba3a9b4e828feaff0cde4e Signed-off-by: mark.j.leonard --- Readme.md | 17 +- pom.xml | 8 +- .../aai/modelloader/config/ModelLoaderConfig.java | 67 +++--- .../onap/aai/modelloader/entity/ArtifactType.java | 3 +- .../entity/catalog/VnfCatalogArtifact.java | 1 + .../entity/catalog/VnfCatalogArtifactHandler.java | 190 ++++++++++++---- .../extraction/VnfCatalogExtractor.java | 107 +++++++++ .../notification/ArtifactDownloadManager.java | 36 ++- .../notification/NotificationPublisher.java | 2 +- .../aai/modelloader/service/ModelLoaderMsgs.java | 245 +++++++++++---------- .../aai/modelloader/util/JsonXmlConverter.java | 4 +- .../modelloader/service/ModelLoaderMsgs.properties | 6 + .../csar/extractor/VnfCatalogExtractorTest.java | 138 ++++++++++++ .../catalog/TestVnfCatalogArtifactHandler.java | 49 +++++ .../ArtifactDownloadManagerVnfcTest.java | 227 +++++++++++++++++++ .../service/TestModelLoaderService.java | 4 +- .../aai/modelloader/util/TestJsonXmlConverter.java | 36 +-- .../compressedArtifacts/noVnfcFilesArchive.csar | Bin 0 -> 1193 bytes .../compressedArtifacts/threeVnfcFilesArchive.csar | Bin 0 -> 3501 bytes src/test/resources/xmlFiles/fortigate.xml | 129 +++++++++++ src/test/resources/xmlFiles/vnfcatalog-1.xml | 100 +++++++++ src/test/resources/xmlFiles/vnfcatalog-2.xml | 100 +++++++++ src/test/resources/xmlFiles/vnfcatalog-3.xml | 100 +++++++++ 23 files changed, 1339 insertions(+), 230 deletions(-) create mode 100644 src/main/java/org/onap/aai/modelloader/extraction/VnfCatalogExtractor.java create mode 100644 src/test/java/org/onap/aai/modelloader/csar/extractor/VnfCatalogExtractorTest.java create mode 100644 src/test/java/org/onap/aai/modelloader/notification/ArtifactDownloadManagerVnfcTest.java create mode 100644 src/test/resources/compressedArtifacts/noVnfcFilesArchive.csar create mode 100644 src/test/resources/compressedArtifacts/threeVnfcFilesArchive.csar create mode 100644 src/test/resources/xmlFiles/fortigate.xml create mode 100644 src/test/resources/xmlFiles/vnfcatalog-1.xml create mode 100644 src/test/resources/xmlFiles/vnfcatalog-2.xml create mode 100644 src/test/resources/xmlFiles/vnfcatalog-3.xml diff --git a/Readme.md b/Readme.md index d8f1f02..3d7aaae 100644 --- a/Readme.md +++ b/Readme.md @@ -1,7 +1,7 @@ # Introduction The A&AI Model Loader Service is an application that facilitates the distribution and ingestion of -new service and resource models from the SDC to the A&AI. +new service and resource models and VNF catalogs from the SDC to the A&AI. ## Features @@ -11,7 +11,18 @@ The Model Loader: * polls the UEB/DMaap cluster for notification events * downloads artifacts from SDC upon receipt of a distribution event * pushes distribution components to A&AI + +### VNF Catalog loading +The Model Loader supports two methods for supplying VNF Catalog data for loading into A&AI: + +* Embedded TOSCA image and vendor data
VNF Catalog data can be embedded within the TOSCA yaml files contained in the CSAR. + + +* VNF Catalog XML files
VNF Catalog data in the form of XML files can be supplied in the CSAR under the path `Artifacts/Deployment/VNF_CATALOG` + +**Note: Each CSAR should provide VNF Catalog information using only one of the above methods. If a CSAR contains both TOSCA and XML VNF Catalog information, a deploy failure will be logged and published to SDC, and no VNF Catalog data will be loaded into A&AI** + ## Compiling Model Loader Model Loader can be compiled by running `mvn clean install` @@ -35,7 +46,7 @@ You will be mounting these as data volumes when you start the Docker container. The following file must be present in this directory on the host machine: -_model-loader.properties_ +_model-loader.properties_ # Always false. TLS Auth currently not supported ml.distribution.ACTIVE_SERVER_TLS_AUTH=false @@ -127,6 +138,6 @@ You can now start the Docker container for the _Model Loader Service_, e.g: {{your docker repo}}/model-loader where - + {{your docker repo}} is the Docker repository you have published your image to. diff --git a/pom.xml b/pom.xml index ae5e7cd..16c478c 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ org.onap.oparent oparent - 1.1.0 + 1.1.1 @@ -52,6 +52,7 @@ ${basedir}/target 1.5.12.RELEASE 1.1 + 1.14 1.3 1.2.0 1.2.1 @@ -151,6 +152,11 @@ jline 2.12.1 + + org.apache.commons + commons-compress + ${commons-compress.version} + commons-io commons-io diff --git a/src/main/java/org/onap/aai/modelloader/config/ModelLoaderConfig.java b/src/main/java/org/onap/aai/modelloader/config/ModelLoaderConfig.java index 0a2bc85..01505ba 100644 --- a/src/main/java/org/onap/aai/modelloader/config/ModelLoaderConfig.java +++ b/src/main/java/org/onap/aai/modelloader/config/ModelLoaderConfig.java @@ -21,6 +21,8 @@ package org.onap.aai.modelloader.config; import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -47,29 +49,29 @@ public class ModelLoaderConfig implements IConfiguration { private static final String SUFFIX_TRUSTSTORE_PASS = "TRUSTSTORE_PASSWORD"; // Configuration file properties - protected static final String PROP_ML_DISTRIBUTION_ACTIVE_SERVER_TLS_AUTH = PREFIX_DISTRIBUTION_CLIENT - + "ACTIVE_SERVER_TLS_AUTH"; - protected static final String PROP_ML_DISTRIBUTION_ASDC_CONNECTION_DISABLED = PREFIX_DISTRIBUTION_CLIENT - + "ASDC_CONNECTION_DISABLE"; + protected static final String PROP_ML_DISTRIBUTION_ACTIVE_SERVER_TLS_AUTH = + PREFIX_DISTRIBUTION_CLIENT + "ACTIVE_SERVER_TLS_AUTH"; + protected static final String PROP_ML_DISTRIBUTION_ASDC_CONNECTION_DISABLED = + PREFIX_DISTRIBUTION_CLIENT + "ASDC_CONNECTION_DISABLE"; protected static final String PROP_ML_DISTRIBUTION_ASDC_ADDRESS = PREFIX_DISTRIBUTION_CLIENT + "ASDC_ADDRESS"; protected static final String PROP_ML_DISTRIBUTION_CONSUMER_GROUP = PREFIX_DISTRIBUTION_CLIENT + "CONSUMER_GROUP"; protected static final String PROP_ML_DISTRIBUTION_CONSUMER_ID = PREFIX_DISTRIBUTION_CLIENT + "CONSUMER_ID"; - protected static final String PROP_ML_DISTRIBUTION_ENVIRONMENT_NAME = PREFIX_DISTRIBUTION_CLIENT - + "ENVIRONMENT_NAME"; - protected static final String PROP_ML_DISTRIBUTION_KEYSTORE_PASSWORD = PREFIX_DISTRIBUTION_CLIENT - + SUFFIX_KEYSTORE_PASS; - protected static final String PROP_ML_DISTRIBUTION_KEYSTORE_FILE = PREFIX_DISTRIBUTION_CLIENT - + SUFFIX_KEYSTORE_FILE; + protected static final String PROP_ML_DISTRIBUTION_ENVIRONMENT_NAME = + PREFIX_DISTRIBUTION_CLIENT + "ENVIRONMENT_NAME"; + protected static final String PROP_ML_DISTRIBUTION_KEYSTORE_PASSWORD = + PREFIX_DISTRIBUTION_CLIENT + SUFFIX_KEYSTORE_PASS; + protected static final String PROP_ML_DISTRIBUTION_KEYSTORE_FILE = + PREFIX_DISTRIBUTION_CLIENT + SUFFIX_KEYSTORE_FILE; protected static final String PROP_ML_DISTRIBUTION_PASSWORD = PREFIX_DISTRIBUTION_CLIENT + "PASSWORD"; - protected static final String PROP_ML_DISTRIBUTION_POLLING_INTERVAL = PREFIX_DISTRIBUTION_CLIENT - + "POLLING_INTERVAL"; + protected static final String PROP_ML_DISTRIBUTION_POLLING_INTERVAL = + PREFIX_DISTRIBUTION_CLIENT + "POLLING_INTERVAL"; protected static final String PROP_ML_DISTRIBUTION_POLLING_TIMEOUT = PREFIX_DISTRIBUTION_CLIENT + "POLLING_TIMEOUT"; protected static final String PROP_ML_DISTRIBUTION_USER = PREFIX_DISTRIBUTION_CLIENT + "USER"; protected static final String PROP_ML_DISTRIBUTION_ARTIFACT_TYPES = PREFIX_DISTRIBUTION_CLIENT + "ARTIFACT_TYPES"; - protected static final String PROP_ML_DISTRIBUTION_MSG_BUS_ADDRESSES = PREFIX_DISTRIBUTION_CLIENT - + "MSG_BUS_ADDRESSES"; - protected static final String PROP_ML_DISTRIBUTION_HTTPS_WITH_DMAAP = PREFIX_DISTRIBUTION_CLIENT - + "USE_HTTPS_WITH_DMAAP"; + protected static final String PROP_ML_DISTRIBUTION_MSG_BUS_ADDRESSES = + PREFIX_DISTRIBUTION_CLIENT + "MSG_BUS_ADDRESSES"; + protected static final String PROP_ML_DISTRIBUTION_HTTPS_WITH_DMAAP = + PREFIX_DISTRIBUTION_CLIENT + "USE_HTTPS_WITH_DMAAP"; protected static final String PROP_AAI_BASE_URL = PREFIX_AAI + "BASE_URL"; protected static final String PROP_AAI_KEYSTORE_FILE = PREFIX_AAI + SUFFIX_KEYSTORE_FILE; @@ -88,29 +90,16 @@ public class ModelLoaderConfig implements IConfiguration { protected static final String PROP_BABEL_GENERATE_RESOURCE_URL = PREFIX_BABEL + "GENERATE_ARTIFACTS_URL"; protected static final String PROP_DEBUG_INGEST_SIMULATOR = PREFIX_DEBUG + "INGEST_SIMULATOR"; - private static String configHome; + protected static final String FILESEP = + (System.getProperty("file.separator") == null) ? "/" : System.getProperty("file.separator"); + private static String configHome; private Properties modelLoaderProperties = null; - private String certLocation = "."; - private List artifactTypes = null; - private List msgBusAddrs = null; - private String modelVersion = null; - protected static final String FILESEP = (System.getProperty("file.separator") == null) ? "/" - : System.getProperty("file.separator"); - - public static void setConfigHome(String configHome) { - ModelLoaderConfig.configHome = configHome; - } - - public static String propertiesFile() { - return configHome + FILESEP + "model-loader.properties"; - } - public ModelLoaderConfig(Properties configProperties) { this(configProperties, ModelLoaderConfig.configHome + FILESEP + "auth" + FILESEP); } @@ -118,10 +107,8 @@ public class ModelLoaderConfig implements IConfiguration { /** * Original constructor * - * @param modelLoaderProperties - * properties needed to be configured for the model loader - * @param certLocation - * location of the certificate + * @param modelLoaderProperties properties needed to be configured for the model loader + * @param certLocation location of the certificate */ public ModelLoaderConfig(Properties modelLoaderProperties, String certLocation) { this.modelLoaderProperties = modelLoaderProperties; @@ -146,6 +133,14 @@ public class ModelLoaderConfig implements IConfiguration { } } + public static void setConfigHome(String configHome) { + ModelLoaderConfig.configHome = configHome; + } + + public static Path propertiesFile() { + return Paths.get(configHome, "model-loader.properties"); + } + @Override public boolean activateServerTLSAuth() { String value = modelLoaderProperties.getProperty(PROP_ML_DISTRIBUTION_ACTIVE_SERVER_TLS_AUTH); diff --git a/src/main/java/org/onap/aai/modelloader/entity/ArtifactType.java b/src/main/java/org/onap/aai/modelloader/entity/ArtifactType.java index a58c874..587b474 100644 --- a/src/main/java/org/onap/aai/modelloader/entity/ArtifactType.java +++ b/src/main/java/org/onap/aai/modelloader/entity/ArtifactType.java @@ -24,5 +24,6 @@ public enum ArtifactType { MODEL, MODEL_V8, NAMED_QUERY, - VNF_CATALOG; + VNF_CATALOG, + VNF_CATALOG_XML; } diff --git a/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifact.java b/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifact.java index 9a84cf2..7447726 100644 --- a/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifact.java +++ b/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifact.java @@ -28,6 +28,7 @@ public class VnfCatalogArtifact extends Artifact { public VnfCatalogArtifact(String payload) { this(ArtifactType.VNF_CATALOG, payload); } + public VnfCatalogArtifact(ArtifactType artifactType, String payload) { super(artifactType); setPayload(payload); diff --git a/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifactHandler.java b/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifactHandler.java index 3480c68..c54d7b2 100644 --- a/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifactHandler.java +++ b/src/main/java/org/onap/aai/modelloader/entity/catalog/VnfCatalogArtifactHandler.java @@ -1,5 +1,5 @@ /** - * ============LICENSE_START======================================================= + * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. @@ -22,14 +22,20 @@ package org.onap.aai.modelloader.entity.catalog; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import java.io.UnsupportedEncodingException; +import java.io.StringReader; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.UUID; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.text.StringEscapeUtils; +import org.apache.http.client.utils.URIBuilder; import org.onap.aai.cl.api.Logger; import org.onap.aai.cl.eelf.LoggerFactory; import org.onap.aai.modelloader.config.ModelLoaderConfig; @@ -38,7 +44,11 @@ import org.onap.aai.modelloader.entity.ArtifactHandler; import org.onap.aai.modelloader.restclient.AaiRestClient; import org.onap.aai.modelloader.service.ModelLoaderMsgs; import org.onap.aai.restclient.client.OperationResult; -import org.springframework.web.util.UriUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; /** * VNF Catalog specific handling @@ -80,11 +90,47 @@ public class VnfCatalogArtifactHandler extends ArtifactHandler { return true; } + /* + * If something fails in the middle of ingesting the catalog we want to roll back any changes to the DB + */ + @Override + public void rollback(List completedArtifacts, String distributionId, AaiRestClient aaiClient) { + for (Artifact completedArtifact : completedArtifacts) { + Map data = new Gson().fromJson(completedArtifact.getPayload(), + new TypeToken>() {}.getType()); + String url = config.getAaiBaseUrl() + config.getAaiVnfImageUrl() + "/vnf-image/" + data.get(ATTR_UUID); + // Try to delete the image. If something goes wrong we can't really do anything here + aaiClient.getAndDeleteResource(url, distributionId); + } + } + private void distributeVnfcData(AaiRestClient restClient, String distributionId, Artifact vnfcArtifact, List completedArtifacts) throws VnfImageException { + List> vnfcData; + switch (vnfcArtifact.getType()) { + case VNF_CATALOG: + vnfcData = unmarshallVnfcData(vnfcArtifact); + break; + case VNF_CATALOG_XML: + vnfcData = parseXmlVnfcData(vnfcArtifact); + break; + default: + throw new VnfImageException("Unsupported type " + vnfcArtifact.getType()); + } + distributeVnfcData(restClient, distributionId, completedArtifacts, vnfcData); + } - List> vnfcData = unmarshallVnfcData(vnfcArtifact); - + /** + * Build a VNF image from each of the supplied data items, and distribute to AAI + * + * @param restClient + * @param distributionId + * @param completedArtifacts + * @param vnfcData + * @throws VnfImageException + */ + private void distributeVnfcData(AaiRestClient restClient, String distributionId, List completedArtifacts, + List> vnfcData) throws VnfImageException { for (Map dataItem : vnfcData) { // If an empty dataItem is supplied, do nothing. if (dataItem.isEmpty()) { @@ -92,49 +138,47 @@ public class VnfCatalogArtifactHandler extends ArtifactHandler { continue; } - String urlParams; - StringBuilder imageId = new StringBuilder("vnf image"); - - try { - urlParams = buildUrlImgIdStrings(imageId, dataItem); - } catch (UnsupportedEncodingException e) { - throw new VnfImageException(e); + StringBuilder imageIdBuilder = new StringBuilder("vnf image"); + for (Entry entry : dataItem.entrySet()) { + imageIdBuilder.append(" ").append(entry.getValue()); } + String imageId = imageIdBuilder.toString(); + int resultCode = getVnfImage(restClient, distributionId, imageId, dataItem); - OperationResult tryGet = - restClient.getResource(config.getAaiBaseUrl() + config.getAaiVnfImageUrl() + "?" + urlParams, - distributionId, MediaType.APPLICATION_JSON_TYPE); - - if (tryGet == null) { - throw new VnfImageException(imageId.toString()); - } - - int resultCode = tryGet.getResultCode(); if (resultCode == Response.Status.NOT_FOUND.getStatusCode()) { // This vnf-image is missing, so add it boolean success = putVnfImage(restClient, dataItem, distributionId); - if (!success) { - throw new VnfImageException(imageId.toString()); + if (success) { + completedArtifacts.add(new VnfCatalogArtifact(new Gson().toJson(dataItem))); + logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, imageId + " successfully ingested."); + } else { + throw new VnfImageException(imageId); } - completedArtifacts.add(vnfcArtifact); - logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, imageId + " successfully ingested."); } else if (resultCode == Response.Status.OK.getStatusCode()) { logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, imageId + " already exists. Skipping ingestion."); } else { // if other than 404 or 200, something went wrong - throw new VnfImageException(imageId.toString(), resultCode); + throw new VnfImageException(imageId, resultCode); } } } - private String buildUrlImgIdStrings(StringBuilder imageId, Map dataItem) - throws UnsupportedEncodingException { - StringBuilder urlParams = new StringBuilder(); - for (Entry entry : dataItem.entrySet()) { - urlParams.append(entry.getKey()).append("=").append(UriUtils.encode(entry.getValue(), "UTF-8")).append("&"); - imageId.append(" ").append(entry.getValue()); + private int getVnfImage(AaiRestClient restClient, String distributionId, String imageId, + Map dataItem) throws VnfImageException { + try { + URIBuilder b = new URIBuilder(config.getAaiBaseUrl() + config.getAaiVnfImageUrl()); + for (Entry entry : dataItem.entrySet()) { + b.addParameter(entry.getKey(), entry.getValue()); + } + OperationResult tryGet = + restClient.getResource(b.build().toString(), distributionId, MediaType.APPLICATION_JSON_TYPE); + if (tryGet == null) { + throw new VnfImageException(imageId); + } + return tryGet.getResultCode(); + } catch (URISyntaxException ex) { + throw new VnfImageException(ex); } - return urlParams.deleteCharAt(urlParams.length() - 1).toString(); } private boolean putVnfImage(AaiRestClient restClient, Map dataItem, String distributionId) { @@ -155,19 +199,79 @@ public class VnfCatalogArtifactHandler extends ArtifactHandler { new TypeToken>>() {}.getType()); } - /* - * If something fails in the middle of ingesting the catalog we want to roll back any changes to the DB + /** + * Parse the VNF Catalog XML and transform into Key/Value pairs. + * + * @param vnfcArtifact + * @return VNF Image data in Map form + * @throws VnfImageException */ - @Override - public void rollback(List completedArtifacts, String distributionId, AaiRestClient aaiClient) { - for (Artifact completedArtifact : completedArtifacts) { - List> completedImageData = unmarshallVnfcData(completedArtifact); - for (Map data : completedImageData) { - String url = config.getAaiBaseUrl() + config.getAaiVnfImageUrl() + "/vnf-image/" + data.get(ATTR_UUID); - // Try to delete the image. If something goes wrong we can't really do anything here - aaiClient.getAndDeleteResource(url, distributionId); + private List> parseXmlVnfcData(Artifact vnfcArtifact) throws VnfImageException { + List> vnfcData = new ArrayList<>(); + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder builder = factory.newDocumentBuilder(); + InputSource is = new InputSource(new StringReader(vnfcArtifact.getPayload())); + Document doc = builder.parse(is); + doc.getDocumentElement().normalize(); + + NodeList pnl = doc.getElementsByTagName("part-number-list"); + for (int i = 0; i < pnl.getLength(); i++) { + Node partNumber = pnl.item(i); + if (partNumber.getNodeType() == Node.ELEMENT_NODE) { + Element vendorInfo = getFirstChildNodeByName(partNumber, "vendor-info"); + if (vendorInfo != null) { + Map application = new HashMap<>(); + application.put("application", + vendorInfo.getElementsByTagName("vendor-model").item(0).getTextContent()); + application.put("application-vendor", + vendorInfo.getElementsByTagName("vendor-name").item(0).getTextContent()); + populateSoftwareVersions(vnfcData, application, partNumber); + } + } + } + } catch (Exception ex) { + throw new VnfImageException(ex); + } + return vnfcData; + } + + /** + * @param vnfcData to populate + * @param applicationData + * @param partNumber + */ + private void populateSoftwareVersions(List> vnfcData, Map applicationData, + Node partNumber) { + NodeList nodes = partNumber.getChildNodes(); + for (int i = 0; i < nodes.getLength(); i++) { + Node childNode = nodes.item(i); + if (childNode.getNodeName().equalsIgnoreCase("software-version-list")) { + Element softwareVersion = getFirstChildNodeByName(childNode, "software-version"); + if (softwareVersion != null) { + HashMap vnfImageData = new HashMap<>(applicationData); + vnfImageData.put("application-version", softwareVersion.getTextContent()); + vnfcData.add(vnfImageData); + } + } + } + } + + /** + * @param node + * @param childNodeName + * @return the first child node matching the given name + */ + private Element getFirstChildNodeByName(Node node, String childNodeName) { + NodeList nodes = node.getChildNodes(); + for (int i = 0; i < nodes.getLength(); i++) { + Node childNode = nodes.item(i); + if (childNode.getNodeName().equalsIgnoreCase(childNodeName)) { + return (Element) childNode; } } + return null; } } diff --git a/src/main/java/org/onap/aai/modelloader/extraction/VnfCatalogExtractor.java b/src/main/java/org/onap/aai/modelloader/extraction/VnfCatalogExtractor.java new file mode 100644 index 0000000..5682774 --- /dev/null +++ b/src/main/java/org/onap/aai/modelloader/extraction/VnfCatalogExtractor.java @@ -0,0 +1,107 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 European Software Marketing Ltd. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.modelloader.extraction; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.regex.Pattern; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.onap.aai.cl.api.Logger; +import org.onap.aai.cl.eelf.LoggerFactory; +import org.onap.aai.modelloader.entity.Artifact; +import org.onap.aai.modelloader.entity.ArtifactType; +import org.onap.aai.modelloader.entity.catalog.VnfCatalogArtifact; +import org.onap.aai.modelloader.service.ModelLoaderMsgs; + + +/** + * The purpose of this class is to process a .csar file in the form of a byte array and extract VNF Catalog XML files + * from it. + * + * A .csar file is a compressed archive like a zip file and this class will treat the byte array as it if were a zip + * file. + */ +public class VnfCatalogExtractor { + private static final Logger logger = LoggerFactory.getInstance().getLogger(VnfCatalogExtractor.class.getName()); + + private static final Pattern VNFCFILE_EXTENSION_REGEX = + Pattern.compile("(?i)artifacts(\\\\|\\/)deployment(\\\\|\\/)vnf_catalog(\\\\|\\/).*\\.xml$"); + + /** + * This method is responsible for filtering the contents of the supplied archive and returning a collection of + * {@link Artifact}s that represent the VNF Catalog files that have been found in the archive.
+ *
+ * If the archive contains no VNF Catalog files it will return an empty list.
+ * + * @param archive the zip file in the form of a byte array containing zero or more VNF Catalog files + * @param name the name of the archive file + * @return List collection of VNF Catalog XML files found in the archive + * @throws InvalidArchiveException if an error occurs trying to extract the VNFC files from the archive or if the + * archive is not a zip file + */ + public List extract(byte[] archive, String name) throws InvalidArchiveException { + validateRequest(archive, name); + + logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, "Extracting CSAR archive: " + name); + + List vnfcFiles = new ArrayList<>(); + try (SeekableInMemoryByteChannel inMemoryByteChannel = new SeekableInMemoryByteChannel(archive); + ZipFile zipFile = new ZipFile(inMemoryByteChannel)) { + for (Enumeration enumeration = zipFile.getEntries(); enumeration.hasMoreElements();) { + ZipArchiveEntry entry = enumeration.nextElement(); + if (fileShouldBeExtracted(entry)) { + vnfcFiles.add(new VnfCatalogArtifact(ArtifactType.VNF_CATALOG_XML, + IOUtils.toString(zipFile.getInputStream(entry)))); + } + } + } catch (IOException e) { + throw new InvalidArchiveException( + "An error occurred trying to create a ZipFile. Is the content being converted really a csar file?", + e); + } + + logger.debug(ModelLoaderMsgs.DISTRIBUTION_EVENT, vnfcFiles.size() + " VNF Catalog files extracted."); + + return vnfcFiles; + } + + private static void validateRequest(byte[] archive, String name) throws InvalidArchiveException { + if (archive == null || archive.length == 0) { + throw new InvalidArchiveException("An archive must be supplied for processing."); + } else if (StringUtils.isBlank(name)) { + throw new InvalidArchiveException("The name of the archive must be supplied for processing."); + } + } + + private static boolean fileShouldBeExtracted(ZipArchiveEntry entry) { + logger.debug(ModelLoaderMsgs.DISTRIBUTION_EVENT, "Checking if " + entry.getName() + " should be extracted..."); + boolean extractFile = VNFCFILE_EXTENSION_REGEX.matcher(entry.getName()).matches(); + logger.debug(ModelLoaderMsgs.DISTRIBUTION_EVENT, "Keeping file: " + entry.getName() + "? : " + extractFile); + return extractFile; + } +} + diff --git a/src/main/java/org/onap/aai/modelloader/notification/ArtifactDownloadManager.java b/src/main/java/org/onap/aai/modelloader/notification/ArtifactDownloadManager.java index febc99a..58bb074 100644 --- a/src/main/java/org/onap/aai/modelloader/notification/ArtifactDownloadManager.java +++ b/src/main/java/org/onap/aai/modelloader/notification/ArtifactDownloadManager.java @@ -1,5 +1,5 @@ /** - * ============LICENSE_START======================================================= + * ============LICENSE_START======================================================= * org.onap.aai * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. @@ -38,6 +38,7 @@ import org.onap.aai.modelloader.entity.model.BabelArtifactParsingException; import org.onap.aai.modelloader.entity.model.IModelParser; import org.onap.aai.modelloader.entity.model.NamedQueryArtifactParser; import org.onap.aai.modelloader.extraction.InvalidArchiveException; +import org.onap.aai.modelloader.extraction.VnfCatalogExtractor; import org.onap.aai.modelloader.restclient.BabelServiceClient; import org.onap.aai.modelloader.restclient.BabelServiceClientException; import org.onap.aai.modelloader.service.BabelServiceClientFactory; @@ -68,6 +69,7 @@ public class ArtifactDownloadManager { private BabelArtifactConverter babelArtifactConverter; private ModelLoaderConfig config; private BabelServiceClientFactory clientFactory; + private VnfCatalogExtractor vnfCatalogExtractor; public ArtifactDownloadManager(IDistributionClient client, ModelLoaderConfig config, BabelServiceClientFactory clientFactory) { @@ -148,13 +150,33 @@ public class ArtifactDownloadManager { } public void processToscaArtifacts(List modelArtifacts, List catalogArtifacts, byte[] payload, + IArtifactInfo artifactInfo, String distributionId, String serviceVersion) + throws ProcessToscaArtifactsException, InvalidArchiveException { + // Get translated artifacts from Babel Service + invokeBabelService(modelArtifacts, catalogArtifacts, payload, artifactInfo, distributionId, serviceVersion); + + // Get VNF Catalog artifacts directly from CSAR + List csarCatalogArtifacts = getVnfCatalogExtractor().extract(payload, artifactInfo.getArtifactName()); + + // Throw an error if VNF Catalog data is present in the Babel payload and directly in the CSAR + if (!catalogArtifacts.isEmpty() && !csarCatalogArtifacts.isEmpty()) { + logger.error(ModelLoaderMsgs.DUPLICATE_VNFC_DATA_ERROR, artifactInfo.getArtifactName()); + throw new InvalidArchiveException("CSAR: " + artifactInfo.getArtifactName() + + " contains VNF Catalog data in the format of both TOSCA and XML files. Only one format can be used for each CSAR file."); + } else if (!csarCatalogArtifacts.isEmpty()) { + catalogArtifacts.addAll(csarCatalogArtifacts); + } + } + + public void invokeBabelService(List modelArtifacts, List catalogArtifacts, byte[] payload, IArtifactInfo artifactInfo, String distributionId, String serviceVersion) throws ProcessToscaArtifactsException { try { BabelServiceClient babelClient = createBabelServiceClient(artifactInfo, serviceVersion); - logger.debug(ModelLoaderMsgs.DISTRIBUTION_EVENT, - "Posting artifact: " + artifactInfo.getArtifactName() + ", version: " + serviceVersion); + logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, + "Posting artifact: " + artifactInfo.getArtifactName() + ", service version: " + serviceVersion + + ", artifact version: " + artifactInfo.getArtifactVersion()); List babelArtifacts = babelClient.postArtifact(payload, artifactInfo.getArtifactName(), serviceVersion, distributionId); @@ -251,4 +273,12 @@ public class ArtifactDownloadManager { return babelArtifactConverter; } + + private VnfCatalogExtractor getVnfCatalogExtractor() { + if (vnfCatalogExtractor == null) { + vnfCatalogExtractor = new VnfCatalogExtractor(); + } + + return vnfCatalogExtractor; + } } diff --git a/src/main/java/org/onap/aai/modelloader/notification/NotificationPublisher.java b/src/main/java/org/onap/aai/modelloader/notification/NotificationPublisher.java index 626399e..4f552f7 100644 --- a/src/main/java/org/onap/aai/modelloader/notification/NotificationPublisher.java +++ b/src/main/java/org/onap/aai/modelloader/notification/NotificationPublisher.java @@ -52,7 +52,7 @@ public class NotificationPublisher { public NotificationPublisher() { Properties configProperties = new Properties(); try { - configProperties.load(Files.newInputStream(Paths.get(ModelLoaderConfig.propertiesFile()))); + configProperties.load(Files.newInputStream(ModelLoaderConfig.propertiesFile())); } catch (IOException e) { String errorMsg = "Failed to load configuration: " + e.getMessage(); logger.error(ModelLoaderMsgs.DISTRIBUTION_EVENT_ERROR, e, errorMsg); diff --git a/src/main/java/org/onap/aai/modelloader/service/ModelLoaderMsgs.java b/src/main/java/org/onap/aai/modelloader/service/ModelLoaderMsgs.java index fd39a9a..26a9efa 100644 --- a/src/main/java/org/onap/aai/modelloader/service/ModelLoaderMsgs.java +++ b/src/main/java/org/onap/aai/modelloader/service/ModelLoaderMsgs.java @@ -1,120 +1,125 @@ -/** - * ============LICENSE_START======================================================= - * org.onap.aai - * ================================================================================ - * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. - * Copyright © 2017-2018 European Software Marketing Ltd. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============LICENSE_END========================================================= - */ -package org.onap.aai.modelloader.service; - -import com.att.eelf.i18n.EELFResourceManager; -import org.onap.aai.cl.eelf.LogMessageEnum; - -public enum ModelLoaderMsgs implements LogMessageEnum { - - /** - * Arguments: None. - */ - LOADING_CONFIGURATION, - - /** - * Arguments: None. - */ - STOPPING_CLIENT, - - /** - * Arguments: {0} = message. - */ - INITIALIZING, - - /** - * Arguments: {0} = reason. - */ - ASDC_CONNECTION_ERROR, - - /** - * Arguments: {0} = message. - */ - DISTRIBUTION_EVENT, - - /** - * Arguments: {0} = error message. - */ - DISTRIBUTION_EVENT_ERROR, - - /** - * Arguments: {0} = request type. {1} = endpoint. {2} = result code. - */ - AAI_REST_REQUEST_SUCCESS, - - /** - * Arguments: {0} = request type. {1} = endpoint. {2} = result code. {3} = result. message - */ - AAI_REST_REQUEST_UNSUCCESSFUL, - - /** - * Arguments: {0} = request type. {1} = endpoint. {2} = error message. - */ - AAI_REST_REQUEST_ERROR, - - /** - * Arguments: {0} = request type. {1} = endpoint. {2} = error message. - */ - BABEL_REST_REQUEST_ERROR, - - /** - * Arguments: {0} = info request payload. - **/ - AAI_REST_REQUEST_PAYLOAD, - - /** - * Arguments: {0} = artifact name - */ - ARTIFACT_PARSE_ERROR, - - /** - * Arguments: {0} = info request payload. - **/ - BABEL_REST_REQUEST_PAYLOAD, - - /** - * Arguments: {0} = info Babel response payload. - **/ - BABEL_RESPONSE_PAYLOAD, - - /** - * Arguments: {0} = artifact name. {1} = payload. - */ - DOWNLOAD_COMPLETE, - - /** - * Arguments: {0} = event. {1} = artifact name. {2} = result. - */ - EVENT_PUBLISHED, - - /** - * Arguments: {0} = artifact name. {1} = artifact type. - */ - UNSUPPORTED_ARTIFACT_TYPE; - - /** - * Load message bundle (ModelLoaderMsgs.properties file) - */ - static { - EELFResourceManager.loadMessageBundle("org/onap/aai/modelloader/service/ModelLoaderMsgs"); - } - -} +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 European Software Marketing Ltd. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.modelloader.service; + +import com.att.eelf.i18n.EELFResourceManager; +import org.onap.aai.cl.eelf.LogMessageEnum; + +public enum ModelLoaderMsgs implements LogMessageEnum { + + /** + * Arguments: None. + */ + LOADING_CONFIGURATION, + + /** + * Arguments: None. + */ + STOPPING_CLIENT, + + /** + * Arguments: {0} = message. + */ + INITIALIZING, + + /** + * Arguments: {0} = reason. + */ + ASDC_CONNECTION_ERROR, + + /** + * Arguments: {0} = message. + */ + DISTRIBUTION_EVENT, + + /** + * Arguments: {0} = error message. + */ + DISTRIBUTION_EVENT_ERROR, + + /** + * Arguments: {0} = request type. {1} = endpoint. {2} = result code. + */ + AAI_REST_REQUEST_SUCCESS, + + /** + * Arguments: {0} = request type. {1} = endpoint. {2} = result code. {3} = result. message + */ + AAI_REST_REQUEST_UNSUCCESSFUL, + + /** + * Arguments: {0} = request type. {1} = endpoint. {2} = error message. + */ + AAI_REST_REQUEST_ERROR, + + /** + * Arguments: {0} = request type. {1} = endpoint. {2} = error message. + */ + BABEL_REST_REQUEST_ERROR, + + /** + * Arguments: {0} = info request payload. + **/ + AAI_REST_REQUEST_PAYLOAD, + + /** + * Arguments: {0} = artifact name + */ + ARTIFACT_PARSE_ERROR, + + /** + * Arguments: {0} = info request payload. + **/ + BABEL_REST_REQUEST_PAYLOAD, + + /** + * Arguments: {0} = info Babel response payload. + **/ + BABEL_RESPONSE_PAYLOAD, + + /** + * Arguments: {0} = artifact name. {1} = payload. + */ + DOWNLOAD_COMPLETE, + + /** + * Arguments: {0} = event. {1} = artifact name. {2} = result. + */ + EVENT_PUBLISHED, + + /** + * Arguments: {0} = artifact name. {1} = artifact type. + */ + UNSUPPORTED_ARTIFACT_TYPE, + + /** + * Arguments: {0} = artifact name. + */ + DUPLICATE_VNFC_DATA_ERROR; + + /** + * Load message bundle (ModelLoaderMsgs.properties file) + */ + static { + EELFResourceManager.loadMessageBundle("org/onap/aai/modelloader/service/ModelLoaderMsgs"); + } + +} diff --git a/src/main/java/org/onap/aai/modelloader/util/JsonXmlConverter.java b/src/main/java/org/onap/aai/modelloader/util/JsonXmlConverter.java index 87cfdcc..84218e5 100644 --- a/src/main/java/org/onap/aai/modelloader/util/JsonXmlConverter.java +++ b/src/main/java/org/onap/aai/modelloader/util/JsonXmlConverter.java @@ -42,11 +42,11 @@ public final class JsonXmlConverter { try { new JSONObject(text); isValid = true; - } catch (JSONException ex) { + } catch (JSONException ex) { // NOSONAR try { new JSONArray(text); isValid = true; - } catch (JSONException ex1) { + } catch (JSONException ex1) { // NOSONAR isValid = false; } } diff --git a/src/main/resources/org/onap/aai/modelloader/service/ModelLoaderMsgs.properties b/src/main/resources/org/onap/aai/modelloader/service/ModelLoaderMsgs.properties index 1e5658c..d337915 100644 --- a/src/main/resources/org/onap/aai/modelloader/service/ModelLoaderMsgs.properties +++ b/src/main/resources/org/onap/aai/modelloader/service/ModelLoaderMsgs.properties @@ -129,6 +129,12 @@ BABEL_REST_REQUEST_ERROR=\ Failed to send {0} request to {1}: {2}|\ Check configuration. Check network connection to Babel.|\ A failure occurred attempting to send a request to the Babel|\ + +DUPLICATE_VNFC_DATA_ERROR=\ + MDLSVC2006E|\ + Artifact: {0} contains both TOSCA and XML VNF Catalog data.|\ + Check artifact. |\ + A failure occurred attempting to process VNF Catalog data in artifact from SDC|\ # DEBUG Level Logs INITIALIZING=\ diff --git a/src/test/java/org/onap/aai/modelloader/csar/extractor/VnfCatalogExtractorTest.java b/src/test/java/org/onap/aai/modelloader/csar/extractor/VnfCatalogExtractorTest.java new file mode 100644 index 0000000..bd8c33f --- /dev/null +++ b/src/test/java/org/onap/aai/modelloader/csar/extractor/VnfCatalogExtractorTest.java @@ -0,0 +1,138 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 European Software Marketing Ltd. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.modelloader.csar.extractor; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.Test; +import org.onap.aai.modelloader.entity.Artifact; +import org.onap.aai.modelloader.entity.ArtifactType; +import org.onap.aai.modelloader.extraction.InvalidArchiveException; +import org.onap.aai.modelloader.extraction.VnfCatalogExtractor; +import org.onap.aai.modelloader.util.ArtifactTestUtils; + + +/** + * Tests {@link VnfCatalogExtractor}. + */ +public class VnfCatalogExtractorTest { + + private static final String FOO = "foo"; + private static final String SOME_BYTES = "just some bytes that will pass the firsts validation"; + private static final String SUPPLY_AN_ARCHIVE = "An archive must be supplied for processing."; + private static final String SUPPLY_NAME = "The name of the archive must be supplied for processing."; + + @Test + public void nullContentSupplied() { + invalidArgumentsTest(null, FOO, SUPPLY_AN_ARCHIVE); + } + + @Test + public void emptyContentSupplied() { + invalidArgumentsTest(new byte[0], FOO, SUPPLY_AN_ARCHIVE); + } + + @Test + public void nullNameSupplied() { + invalidArgumentsTest(SOME_BYTES.getBytes(), null, SUPPLY_NAME); + } + + @Test + public void blankNameSupplied() { + invalidArgumentsTest("just some bytes that will pass the firsts validation".getBytes(), " \t ", SUPPLY_NAME); + } + + @Test + public void emptyNameSupplied() { + invalidArgumentsTest("just some bytes that will pass the firsts validation".getBytes(), "", SUPPLY_NAME); + } + + @Test + public void invalidContentSupplied() { + invalidArgumentsTest("This is a piece of nonsense and not a zip file".getBytes(), FOO, + "An error occurred trying to create a ZipFile. Is the content being converted really a csar file?"); + } + + @Test + public void archiveContainsNoVnfcFiles() throws InvalidArchiveException, IOException { + List vnfcArtifacts = new VnfCatalogExtractor().extract( + new ArtifactTestUtils().loadResource("compressedArtifacts/noVnfcFilesArchive.csar"), + "noVnfcFilesArchive.csar"); + assertTrue("No VNFC files should have been extracted, but " + vnfcArtifacts.size() + " were found.", + vnfcArtifacts.isEmpty()); + } + + @Test + public void archiveContainsThreeRelevantVnfcFiles() throws IOException, InvalidArchiveException { + List payloads = new ArrayList<>(); + payloads.add("xmlFiles/vnfcatalog-1.xml"); + payloads.add("xmlFiles/vnfcatalog-2.xml"); + payloads.add("xmlFiles/vnfcatalog-3.xml"); + String csarArchiveFile = "threeVnfcFilesArchive.csar"; + performVnfcAsserts(new VnfCatalogExtractor().extract( + new ArtifactTestUtils().loadResource("compressedArtifacts/" + csarArchiveFile), csarArchiveFile), + payloads); + } + + public void performVnfcAsserts(List actualVnfcArtifacts, List expectedVnfcPayloadsToLoad) { + assertThat("An unexpected number of VNFC files have been extracted", actualVnfcArtifacts.size(), + is(expectedVnfcPayloadsToLoad.size())); + + for (Artifact artifact : actualVnfcArtifacts) { + assertThat("Unexpected artifact type found.", artifact.getType(), is(ArtifactType.VNF_CATALOG_XML)); + } + + Set expectedVnfcPayloads = expectedVnfcPayloadsToLoad.stream().map(s -> { + try { + return ArtifactTestUtils.loadResourceAsString(s); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toSet()); + + Set actualVnfcPayloads = + actualVnfcArtifacts.stream().map(s -> s.getPayload()).collect(Collectors.toSet()); + + assertThat("Unexpected VNF Catalog file(s) found.", expectedVnfcPayloads.containsAll(actualVnfcPayloads), + is(true)); + } + + private void invalidArgumentsTest(byte[] archive, String name, String expectedErrorMessage) { + try { + new VnfCatalogExtractor().extract(archive, name); + fail("An instance of InvalidArchiveException should have been thrown"); + } catch (Exception ex) { + assertTrue(ex instanceof InvalidArchiveException); + assertEquals(expectedErrorMessage, ex.getLocalizedMessage()); + } + } + +} + diff --git a/src/test/java/org/onap/aai/modelloader/entity/catalog/TestVnfCatalogArtifactHandler.java b/src/test/java/org/onap/aai/modelloader/entity/catalog/TestVnfCatalogArtifactHandler.java index b54bb20..cead699 100644 --- a/src/test/java/org/onap/aai/modelloader/entity/catalog/TestVnfCatalogArtifactHandler.java +++ b/src/test/java/org/onap/aai/modelloader/entity/catalog/TestVnfCatalogArtifactHandler.java @@ -41,6 +41,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.onap.aai.modelloader.config.ModelLoaderConfig; import org.onap.aai.modelloader.entity.Artifact; +import org.onap.aai.modelloader.entity.ArtifactType; import org.onap.aai.modelloader.restclient.AaiRestClient; import org.onap.aai.restclient.client.OperationResult; @@ -80,6 +81,40 @@ public class TestVnfCatalogArtifactHandler { assertPutOperationsSucceeded(); } + @Test + public void testUpdateVnfImagesFromXml() throws Exception { + // GET operation + OperationResult mockGetResp = mock(OperationResult.class); + + // @formatter:off + when(mockGetResp.getResultCode()) + .thenReturn(Response.Status.OK.getStatusCode()) + .thenReturn(Response.Status.NOT_FOUND.getStatusCode()) + .thenReturn(Response.Status.NOT_FOUND.getStatusCode()) + .thenReturn(Response.Status.OK.getStatusCode()); + // @formatter:on + + when(mockRestClient.getResource(Mockito.anyString(), Mockito.anyString(), Mockito.any(MediaType.class))) + .thenReturn(mockGetResp); + mockPutOperations(); + + // Example VNF Catalog XML + VnfCatalogArtifactHandler handler = new VnfCatalogArtifactHandler(createConfig()); + assertThat( + handler.pushArtifacts(createVnfCatalogXmlArtifact(), "test", new ArrayList(), mockRestClient), + is(true)); + + // Only two of the VNF images should be pushed + ArgumentCaptor argument = ArgumentCaptor.forClass(String.class); + AaiRestClient client = Mockito.verify(mockRestClient, Mockito.times(2)); + client.putResource(Mockito.anyString(), argument.capture(), Mockito.anyString(), Mockito.any(MediaType.class)); + assertThat(argument.getAllValues().size(), is(2)); + assertThat(argument.getAllValues().get(0), containsString("5.2.5")); + assertThat(argument.getAllValues().get(0), containsString("VM00")); + assertThat(argument.getAllValues().get(1), containsString("5.2.4")); + assertThat(argument.getAllValues().get(1), containsString("VM00")); + } + private ModelLoaderConfig createConfig() { Properties configProperties = new Properties(); try { @@ -106,6 +141,20 @@ public class TestVnfCatalogArtifactHandler { return artifacts; } + /** + * Example VNF Catalog based on VNF_CATALOG XML + * + * @return test Artifacts + * @throws IOException + * @throws UnsupportedEncodingException + */ + private List createVnfCatalogXmlArtifact() throws IOException, UnsupportedEncodingException { + byte[] encoded = Files.readAllBytes(Paths.get("src/test/resources/xmlFiles/fortigate.xml")); + List artifacts = new ArrayList(); + artifacts.add(new VnfCatalogArtifact(ArtifactType.VNF_CATALOG_XML, new String(encoded, "utf-8"))); + return artifacts; + } + /** * Always return CREATED (success) for a PUT operation. */ diff --git a/src/test/java/org/onap/aai/modelloader/notification/ArtifactDownloadManagerVnfcTest.java b/src/test/java/org/onap/aai/modelloader/notification/ArtifactDownloadManagerVnfcTest.java new file mode 100644 index 0000000..6c7a77f --- /dev/null +++ b/src/test/java/org/onap/aai/modelloader/notification/ArtifactDownloadManagerVnfcTest.java @@ -0,0 +1,227 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. + * Copyright © 2017-2018 European Software Marketing Ltd. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.modelloader.notification; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.onap.aai.modelloader.fixture.NotificationDataFixtureBuilder.getNotificationDataWithToscaCsarFile; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.mockito.internal.util.reflection.Whitebox; +import org.onap.aai.babel.service.data.BabelArtifact; +import org.onap.aai.modelloader.config.ModelLoaderConfig; +import org.onap.aai.modelloader.entity.Artifact; +import org.onap.aai.modelloader.entity.ArtifactType; +import org.onap.aai.modelloader.entity.catalog.VnfCatalogArtifact; +import org.onap.aai.modelloader.entity.model.BabelArtifactParsingException; +import org.onap.aai.modelloader.entity.model.ModelArtifact; +import org.onap.aai.modelloader.extraction.InvalidArchiveException; +import org.onap.aai.modelloader.extraction.VnfCatalogExtractor; +import org.onap.aai.modelloader.restclient.BabelServiceClient; +import org.onap.aai.modelloader.restclient.BabelServiceClientException; +import org.onap.aai.modelloader.service.BabelServiceClientFactory; +import org.onap.aai.modelloader.util.ArtifactTestUtils; +import org.onap.sdc.api.IDistributionClient; +import org.onap.sdc.api.notification.IArtifactInfo; +import org.onap.sdc.api.notification.INotificationData; +import org.onap.sdc.api.results.IDistributionClientDownloadResult; +import org.onap.sdc.impl.DistributionClientDownloadResultImpl; +import org.onap.sdc.utils.DistributionActionResultEnum; + +/** + * Tests {@link ArtifactDownloadManager} with VNF Catalog Artifacts. + */ +public class ArtifactDownloadManagerVnfcTest { + + private ArtifactDownloadManager downloadManager; + private BabelServiceClient mockBabelClient; + private IDistributionClient mockDistributionClient; + private NotificationPublisher mockNotificationPublisher; + private BabelArtifactConverter mockBabelArtifactConverter; + private BabelServiceClientFactory mockClientFactory; + private VnfCatalogExtractor mockVnfCatalogExtractor; + + @Before + public void setup() throws Exception { + mockBabelClient = mock(BabelServiceClient.class); + mockDistributionClient = mock(IDistributionClient.class); + mockNotificationPublisher = mock(NotificationPublisher.class); + mockBabelArtifactConverter = mock(BabelArtifactConverter.class); + mockClientFactory = mock(BabelServiceClientFactory.class); + when(mockClientFactory.create(Mockito.any())).thenReturn(mockBabelClient); + mockVnfCatalogExtractor = mock(VnfCatalogExtractor.class); + + Properties configProperties = new Properties(); + configProperties.load(this.getClass().getClassLoader().getResourceAsStream("model-loader.properties")); + downloadManager = new ArtifactDownloadManager(mockDistributionClient, + new ModelLoaderConfig(configProperties, "."), mockClientFactory); + + + Whitebox.setInternalState(downloadManager, "notificationPublisher", mockNotificationPublisher); + Whitebox.setInternalState(downloadManager, "babelArtifactConverter", mockBabelArtifactConverter); + Whitebox.setInternalState(downloadManager, "vnfCatalogExtractor", mockVnfCatalogExtractor); + } + + @Test + public void downloadArtifacts_validToscaVnfcCsarFile() + throws IOException, BabelServiceClientException, BabelArtifactParsingException, InvalidArchiveException { + INotificationData data = getNotificationDataWithToscaCsarFile(); + IArtifactInfo artifactInfo = data.getServiceArtifacts().get(0); + + setupValidDownloadCsarMocks(data, artifactInfo); + when(mockBabelClient.postArtifact(Matchers.any(), Matchers.anyString(), Matchers.anyString(), + Matchers.anyString())).thenReturn(createBabelArtifacts()); + when(mockVnfCatalogExtractor.extract(Matchers.any(), Matchers.anyString())).thenReturn(new ArrayList<>()); + + List modelArtifacts = new ArrayList<>(); + List catalogFiles = new ArrayList<>(); + assertThat(downloadManager.downloadArtifacts(data, data.getServiceArtifacts(), modelArtifacts, catalogFiles), + is(true)); + + assertEquals("There should have been some catalog files", 2, catalogFiles.size()); + } + + @Test + public void downloadArtifacts_validXmlVnfcCsarFile() + throws IOException, BabelServiceClientException, BabelArtifactParsingException, InvalidArchiveException { + INotificationData data = getNotificationDataWithToscaCsarFile(); + IArtifactInfo artifactInfo = data.getServiceArtifacts().get(0); + + setupValidDownloadCsarMocks(data, artifactInfo); + when(mockBabelClient.postArtifact(Matchers.any(), Matchers.anyString(), Matchers.anyString(), + Matchers.anyString())).thenReturn(createBabelArtifactsNoVnfc()); + when(mockVnfCatalogExtractor.extract(Matchers.any(), Matchers.anyString())) + .thenReturn(createXmlVnfcArtifacts()); + + List modelArtifacts = new ArrayList<>(); + List catalogFiles = new ArrayList<>(); + assertThat(downloadManager.downloadArtifacts(data, data.getServiceArtifacts(), modelArtifacts, catalogFiles), + is(true)); + + assertEquals("There should have been some catalog files", 3, catalogFiles.size()); + } + + @Test + public void downloadArtifacts_validNoVnfcCsarFile() + throws IOException, BabelServiceClientException, BabelArtifactParsingException, InvalidArchiveException { + INotificationData data = getNotificationDataWithToscaCsarFile(); + IArtifactInfo artifactInfo = data.getServiceArtifacts().get(0); + + setupValidDownloadCsarMocks(data, artifactInfo); + when(mockBabelClient.postArtifact(Matchers.any(), Matchers.anyString(), Matchers.anyString(), + Matchers.anyString())).thenReturn(createBabelArtifactsNoVnfc()); + when(mockVnfCatalogExtractor.extract(Matchers.any(), Matchers.anyString())).thenReturn(new ArrayList<>()); + + List modelArtifacts = new ArrayList<>(); + List catalogFiles = new ArrayList<>(); + assertThat(downloadManager.downloadArtifacts(data, data.getServiceArtifacts(), modelArtifacts, catalogFiles), + is(true)); + + assertEquals("There should not have been any catalog files", 0, catalogFiles.size()); + } + + @Test + public void downloadArtifacts_invalidXmlAndToscaVnfcCsarFile() + throws IOException, BabelServiceClientException, BabelArtifactParsingException, InvalidArchiveException { + INotificationData data = getNotificationDataWithToscaCsarFile(); + IArtifactInfo artifactInfo = data.getServiceArtifacts().get(0); + + setupValidDownloadCsarMocks(data, artifactInfo); + when(mockBabelClient.postArtifact(Matchers.any(), Matchers.anyString(), Matchers.anyString(), + Matchers.anyString())).thenReturn(createBabelArtifacts()); + when(mockVnfCatalogExtractor.extract(Matchers.any(), Matchers.anyString())) + .thenReturn(createXmlVnfcArtifacts()); + doNothing().when(mockNotificationPublisher).publishDeployFailure(mockDistributionClient, data, artifactInfo); + + List modelArtifacts = new ArrayList<>(); + List catalogFiles = new ArrayList<>(); + assertThat(downloadManager.downloadArtifacts(data, data.getServiceArtifacts(), modelArtifacts, catalogFiles), + is(false)); + + Mockito.verify(mockNotificationPublisher).publishDeployFailure(mockDistributionClient, data, artifactInfo); + } + + private IDistributionClientDownloadResult createDistributionClientDownloadResult( + DistributionActionResultEnum status, String message, byte[] payload) { + IDistributionClientDownloadResult downloadResult = new DistributionClientDownloadResultImpl(status, message); + + ((DistributionClientDownloadResultImpl) downloadResult).setArtifactPayload(payload); + + return downloadResult; + } + + private void setupValidDownloadCsarMocks(INotificationData data, IArtifactInfo artifactInfo) + throws IOException, BabelServiceClientException, BabelArtifactParsingException { + when(mockDistributionClient.download(artifactInfo)) + .thenReturn(createDistributionClientDownloadResult(DistributionActionResultEnum.SUCCESS, null, + new ArtifactTestUtils().loadResource("compressedArtifacts/service-VscpaasTest-csar.csar"))); + when(mockBabelArtifactConverter.convertToModel(Mockito.anyListOf(BabelArtifact.class))) + .thenReturn(createModelArtifacts()); + when(mockBabelArtifactConverter.convertToCatalog(Mockito.anyListOf(BabelArtifact.class))) + .thenReturn(createToscaVnfcArtifacts()); + } + + private List createBabelArtifacts() { + List artifactList = new ArrayList<>(); + artifactList.add(new BabelArtifact("ModelArtifact", BabelArtifact.ArtifactType.MODEL, "Some model payload")); + artifactList.add(new BabelArtifact("VNFCArtifact", BabelArtifact.ArtifactType.VNFCATALOG, "Some VNFC payload")); + return artifactList; + } + + private List createBabelArtifactsNoVnfc() { + List artifactList = new ArrayList<>(); + artifactList.add(new BabelArtifact("ModelArtifact", BabelArtifact.ArtifactType.MODEL, "Some model payload")); + return artifactList; + } + + private List createModelArtifacts() { + List modelArtifacts = new ArrayList<>(); + modelArtifacts.add(new ModelArtifact()); + modelArtifacts.add(new ModelArtifact()); + return modelArtifacts; + } + + private List createToscaVnfcArtifacts() { + List vnfcArtifacts = new ArrayList<>(); + vnfcArtifacts.add(new VnfCatalogArtifact("Some VNFC payload")); + vnfcArtifacts.add(new VnfCatalogArtifact("Some VNFC payload")); + return vnfcArtifacts; + } + + private List createXmlVnfcArtifacts() { + List vnfcArtifacts = new ArrayList<>(); + vnfcArtifacts.add(new VnfCatalogArtifact(ArtifactType.VNF_CATALOG_XML, "Some VNFC payload")); + vnfcArtifacts.add(new VnfCatalogArtifact(ArtifactType.VNF_CATALOG_XML, "Some VNFC payload")); + vnfcArtifacts.add(new VnfCatalogArtifact(ArtifactType.VNF_CATALOG_XML, "Some VNFC payload")); + return vnfcArtifacts; + } +} diff --git a/src/test/java/org/onap/aai/modelloader/service/TestModelLoaderService.java b/src/test/java/org/onap/aai/modelloader/service/TestModelLoaderService.java index 8e07650..e40ca3c 100644 --- a/src/test/java/org/onap/aai/modelloader/service/TestModelLoaderService.java +++ b/src/test/java/org/onap/aai/modelloader/service/TestModelLoaderService.java @@ -56,7 +56,7 @@ public class TestModelLoaderService { public void testMissingConfig() { new ModelLoaderService().start(); } - + @Test public void testLoadModel() { Response response = service.loadModel(""); @@ -80,7 +80,7 @@ public class TestModelLoaderService { public void testIngestModelMissingName() throws IOException { byte[] csarPayload = new ArtifactTestUtils().loadResource("compressedArtifacts/service-VscpaasTest-csar.csar"); Response response = service.ingestModel("", "", Base64.getEncoder().encodeToString(csarPayload)); - assertThat(response.getStatus(), is(Response.Status.OK.getStatusCode())); + assertThat(response.getStatus(), is(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode())); } } diff --git a/src/test/java/org/onap/aai/modelloader/util/TestJsonXmlConverter.java b/src/test/java/org/onap/aai/modelloader/util/TestJsonXmlConverter.java index 4b7db56..81a4833 100644 --- a/src/test/java/org/onap/aai/modelloader/util/TestJsonXmlConverter.java +++ b/src/test/java/org/onap/aai/modelloader/util/TestJsonXmlConverter.java @@ -49,34 +49,34 @@ public class TestJsonXmlConverter { @Test public void testConversion() throws Exception { - byte[] encoded = Files.readAllBytes(Paths.get(XML_MODEL_FILE)); - assertThat(JsonXmlConverter.isValidJson(new String(encoded)), is(false)); - encoded = Files.readAllBytes(Paths.get(JSON_MODEL_FILE)); + byte[] encoded = Files.readAllBytes(Paths.get(XML_MODEL_FILE)); + assertThat(JsonXmlConverter.isValidJson(new String(encoded)), is(false)); + encoded = Files.readAllBytes(Paths.get(JSON_MODEL_FILE)); String originalJson = new String(encoded); assertThat(JsonXmlConverter.isValidJson(originalJson), is(true)); String xmlFromJson = JsonXmlConverter.convertJsonToXml(originalJson); - // Spot check one of the attributes - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document doc = builder.parse(new ByteArrayInputStream(xmlFromJson.getBytes())); - NodeList nodeList = doc.getDocumentElement().getChildNodes(); + // Spot check one of the attributes + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(new ByteArrayInputStream(xmlFromJson.getBytes())); + NodeList nodeList = doc.getDocumentElement().getChildNodes(); - String modelVid = "notFound"; - for (int i = 0; i < nodeList.getLength(); i++) { - Node currentNode = nodeList.item(i); - if (currentNode.getNodeName().equals("model-invariant-id")) { - modelVid = currentNode.getTextContent(); - break; - } + String modelVid = "notFound"; + for (int i = 0; i < nodeList.getLength(); i++) { + Node currentNode = nodeList.item(i); + if (currentNode.getNodeName().equals("model-invariant-id")) { + modelVid = currentNode.getTextContent(); + break; } + } assertThat(modelVid.equals("3d560d81-57d0-438b-a2a1-5334dba0651a"), is(true)); - // Convert the XML form back into JSON - JsonXmlConverter.convertXmlToJson(xmlFromJson); + // Convert the XML form back into JSON + JsonXmlConverter.convertXmlToJson(xmlFromJson); } } diff --git a/src/test/resources/compressedArtifacts/noVnfcFilesArchive.csar b/src/test/resources/compressedArtifacts/noVnfcFilesArchive.csar new file mode 100644 index 0000000..fa2a327 Binary files /dev/null and b/src/test/resources/compressedArtifacts/noVnfcFilesArchive.csar differ diff --git a/src/test/resources/compressedArtifacts/threeVnfcFilesArchive.csar b/src/test/resources/compressedArtifacts/threeVnfcFilesArchive.csar new file mode 100644 index 0000000..fc5080b Binary files /dev/null and b/src/test/resources/compressedArtifacts/threeVnfcFilesArchive.csar differ diff --git a/src/test/resources/xmlFiles/fortigate.xml b/src/test/resources/xmlFiles/fortigate.xml new file mode 100644 index 0000000..16fce07 --- /dev/null +++ b/src/test/resources/xmlFiles/fortigate.xml @@ -0,0 +1,129 @@ + + + FortiGate-VM00 + FW + + FORTINET + FortiGate-VM00 + VM00 + + + 1 + 1 + 1 + + + 1 + GB + 1 + 1 + + + 2 + GB + 2 + 32 + + + 5.2.7 + 0 + IMAGE + 1c59a521885c465004456f74d003726c + test3 + + + 5.2.5 + 1 + IMAGE + C4D2CBE51669796E48623E006782F7DC + test2 + + + 5.2.4 + 2 + IMAGE + 4987E1E743FD641C879E1D3C5D50BCE0 + test1 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + FortiGate-VM00 + FALSE + + + + FortiGate-VM01 + FW + + FORTINET + FortiGate-VM01 + VM01 + + + 1 + 1 + 1 + + + 2 + GB + 1 + 2 + + + 2 + GB + 2 + 32 + + + 5.2.7 + 0 + IMAGE + 1c59a521885c465004456f74d003726c + software file name 3 + + + 5.2.5 + 1 + IMAGE + C4D2CBE51669796E48623E006782F7DC + software file name 2 + + + 5.2.4 + 2 + IMAGE + 4987E1E743FD641C879E1D3C5D50BCE0 + software file name + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license group + FALSE + + + + \ No newline at end of file diff --git a/src/test/resources/xmlFiles/vnfcatalog-1.xml b/src/test/resources/xmlFiles/vnfcatalog-1.xml new file mode 100644 index 0000000..979589c --- /dev/null +++ b/src/test/resources/xmlFiles/vnfcatalog-1.xml @@ -0,0 +1,100 @@ + + + att-part-number1 + vnf-type1 + + vendor-name1 + vendor-part-number1 + vendor-model1 + + + 2 + 2 + 2 + + + 2 + GB + 2 + 2 + + + 50 + GB + 50 + 50 + + + software-version1 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename1 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group1 + TRUE + + + + att-part-number2 + vnf-type2 + + vendor-name2 + vendor-part-number2 + vendor-model2 + + + 2 + 2 + 2 + + + 2 + GB + 2 + 2 + + + 50 + GB + 50 + 50 + + + software-version2 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename2 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group2 + TRUE + + + \ No newline at end of file diff --git a/src/test/resources/xmlFiles/vnfcatalog-2.xml b/src/test/resources/xmlFiles/vnfcatalog-2.xml new file mode 100644 index 0000000..9bcdac3 --- /dev/null +++ b/src/test/resources/xmlFiles/vnfcatalog-2.xml @@ -0,0 +1,100 @@ + + + att-part-number3 + vnf-type3 + + vendor-name3 + vendor-part-number3 + vendor-model3 + + + 2 + 2 + 2 + + + 2 + GB + 2 + 2 + + + 50 + GB + 50 + 50 + + + software-version3 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename3 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group3 + TRUE + + + + att-part-number4 + vnf-type4 + + vendor-name4 + vendor-part-number4 + vendor-model4 + + + 4 + 4 + 4 + + + 4 + GB + 4 + 4 + + + 50 + GB + 50 + 50 + + + software-version4 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename4 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group4 + TRUE + + + \ No newline at end of file diff --git a/src/test/resources/xmlFiles/vnfcatalog-3.xml b/src/test/resources/xmlFiles/vnfcatalog-3.xml new file mode 100644 index 0000000..d420398 --- /dev/null +++ b/src/test/resources/xmlFiles/vnfcatalog-3.xml @@ -0,0 +1,100 @@ + + + att-part-number5 + vnf-type5 + + vendor-name5 + vendor-part-number5 + vendor-model5 + + + 2 + 2 + 2 + + + 2 + GB + 2 + 2 + + + 50 + GB + 50 + 50 + + + software-version5 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename5 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group5 + TRUE + + + + att-part-number6 + vnf-type6 + + vendor-name6 + vendor-part-number6 + vendor-model6 + + + 6 + 6 + 6 + + + 6 + GB + 6 + 6 + + + 50 + GB + 50 + 50 + + + software-version6 + 0 + IMAGE + BE2B249315B4410896099CFD1AE1948C + software-filename6 + + + APPID + + + IPS-IDS + + + URLF + + + Anti-Virus + + + license-assignment-group6 + TRUE + + + \ No newline at end of file -- cgit 1.2.3-korg