Initial OpenECOMP A&AI Model Loader commit

Change-Id: Iae343fa01ecc701919703fb7d61727555371321d
Signed-off-by: Steve Smokowski <ss835w@att.com>

4 files changed, 623 insertions, 0 deletions

diff --git a/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifact.java b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifact.java
new file mode 100644
index 0000000..904dba9
--- /dev/null
+++ b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifact.java
@@ -0,0 +1,60 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * MODEL LOADER SERVICE
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.modelloader.entity.model;
+
+import org.openecomp.modelloader.entity.Artifact;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class ModelArtifact extends Artifact {
+
+  String nameVersionId;
+  Set<String> referencedModelIds = new HashSet<String>();
+
+  public String getNameVersionId() {
+    return nameVersionId;
+  }
+
+  public void setNameVersionId(String nameVersionId) {
+    this.nameVersionId = nameVersionId;
+  }
+
+  public Set<String> getDependentModelIds() {
+    return referencedModelIds;
+  }
+
+  public void addDependentModelId(String dependentModelId) {
+    this.referencedModelIds.add(dependentModelId);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("NameVersId=" + nameVersionId + "(" + getType().toString() + ") ==> ");
+    for (String dep : referencedModelIds) {
+      sb.append(dep + "  ");
+    }
+
+    return sb.toString();
+  }
+
+}
diff --git a/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactHandler.java b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactHandler.java
new file mode 100644
index 0000000..fb269b1
--- /dev/null
+++ b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactHandler.java
@@ -0,0 +1,133 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * MODEL LOADER SERVICE
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.modelloader.entity.model;
+
+import com.sun.jersey.api.client.ClientResponse;
+
+import org.openecomp.cl.api.Logger;
+import org.openecomp.cl.eelf.LoggerFactory;
+import org.openecomp.modelloader.config.ModelLoaderConfig;
+import org.openecomp.modelloader.entity.Artifact;
+import org.openecomp.modelloader.entity.ArtifactHandler;
+import org.openecomp.modelloader.entity.ArtifactType;
+import org.openecomp.modelloader.restclient.AaiRestClient;
+import org.openecomp.modelloader.service.ModelLoaderMsgs;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.core.Response;
+
+public class ModelArtifactHandler extends ArtifactHandler {
+
+  private static Logger logger = LoggerFactory.getInstance()
+      .getLogger(ArtifactHandler.class.getName());
+
+  public ModelArtifactHandler(ModelLoaderConfig config) {
+    super(config);
+  }
+
+  @Override
+  public boolean pushArtifacts(List<Artifact> artifacts, String distributionId) {
+    ModelSorter modelSorter = new ModelSorter();
+    List<Artifact> sortedModelArtifacts = modelSorter.sort(artifacts);
+
+    // Push the ordered list of model artifacts to A&AI. If one fails, we need
+    // to roll back
+    // the changes.
+    List<ModelArtifact> completedModels = new ArrayList<ModelArtifact>();
+    AaiRestClient aaiClient = new AaiRestClient(config);
+
+    for (Artifact art : sortedModelArtifacts) {
+      ModelArtifact model = (ModelArtifact) art;
+      ClientResponse getResponse = aaiClient.getResource(getUrl(model), distributionId,
+          AaiRestClient.MimeType.XML);
+      if ((getResponse == null)
+          || (getResponse.getStatus() != Response.Status.OK.getStatusCode())) {
+        // Only attempt the PUT if the model doesn't already exist
+        ClientResponse putResponse = aaiClient.putResource(getUrl(model), model.getPayload(),
+            distributionId, AaiRestClient.MimeType.XML);
+        if ((putResponse != null)
+            && (putResponse.getStatus() == Response.Status.CREATED.getStatusCode())) {
+          completedModels.add(model);
+          logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, model.getType().toString() + " "
+              + model.getNameVersionId() + " successfully ingested.");
+        } else {
+          logger.error(ModelLoaderMsgs.DISTRIBUTION_EVENT_ERROR,
+              "Ingestion failed for " + model.getType().toString() + " " + model.getNameVersionId()
+                  + ". Rolling back distribution.");
+
+          for (ModelArtifact modelToDelete : completedModels) {
+            // Best effort to delete. Nothing we can do in the event this fails.
+            aaiClient.getAndDeleteResource(getUrl(modelToDelete), distributionId);
+          }
+
+          return false;
+        }
+      } else {
+        logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT, model.getType().toString() + " "
+            + model.getNameVersionId() + " already exists.  Skipping ingestion.");
+      }
+    }
+
+    return true;
+  }
+
+  private String getUrl(ModelArtifact model) {
+    String baseUrl = config.getAaiBaseUrl().trim();
+    String subUrl = null;
+    if (model.getType().equals(ArtifactType.MODEL)) {
+      subUrl = config.getAaiModelUrl().trim();
+    } else {
+      subUrl = config.getAaiNamedQueryUrl().trim();
+    }
+
+    if ((!baseUrl.endsWith("/")) && (!subUrl.startsWith("/"))) {
+      baseUrl = baseUrl + "/";
+    }
+
+    if (baseUrl.endsWith("/") && subUrl.startsWith("/")) {
+      baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
+    }
+
+    if (!subUrl.endsWith("/")) {
+      subUrl = subUrl + "/";
+    }
+
+    String url = baseUrl + subUrl + model.getNameVersionId();
+    return url;
+  }
+
+  /**
+   * This method is used for the test REST interface to load models without an
+   * ASDC.
+   * 
+   * @param payload content of the request
+   */
+  public void loadModelTest(byte[] payload) {
+    List<Artifact> modelArtifacts = new ArrayList<Artifact>();
+    ModelArtifactParser parser = new ModelArtifactParser();
+    modelArtifacts.addAll(parser.parse(payload, "Test-Artifact"));
+    ModelSorter modelSorter = new ModelSorter();
+    List<Artifact> sortedModelArtifacts = modelSorter.sort(modelArtifacts);
+    pushArtifacts(sortedModelArtifacts, "Test-Distribution");
+  }
+}
diff --git a/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactParser.java b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactParser.java
new file mode 100644
index 0000000..625145f
--- /dev/null
+++ b/src/main/java/org/openecomp/modelloader/entity/model/ModelArtifactParser.java
@@ -0,0 +1,197 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * MODEL LOADER SERVICE
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.modelloader.entity.model;
+
+import org.openecomp.cl.api.Logger;
+import org.openecomp.cl.eelf.LoggerFactory;
+import org.openecomp.modelloader.entity.Artifact;
+import org.openecomp.modelloader.entity.ArtifactType;
+import org.openecomp.modelloader.service.ModelLoaderMsgs;
+import org.openecomp.modelloader.util.JsonXmlConverter;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+public class ModelArtifactParser {
+
+  private static String MODELS_ELEMENT = "models";
+  private static String MODEL_ELEMENT = "model";
+  private static String NAMED_QUERIES_ELEMENT = "named-queries";
+  private static String NAMED_QUERY_ELEMENT = "named-query";
+  private static String MODEL_NAME_VERSION_ID = "model-name-version-id";
+  private static String NAMED_QUERY_VERSION_ID = "named-query-uuid";
+  private static String RELATIONSHIP_DATA = "relationship-data";
+  private static String RELATIONSHIP_KEY = "relationship-key";
+  private static String RELATIONSHIP_VALUE = "relationship-value";
+  private static String MODEL_ELEMENT_RELATIONSHIP_KEY = "model.model-name-version-id";
+
+  private static Logger logger = LoggerFactory.getInstance()
+      .getLogger(ModelArtifactParser.class.getName());
+
+  /**
+   * This method parses the given artifact payload in byte array format and
+   * generates a list of model artifacts according to the content.
+   * 
+   * @param artifactPayload
+   *          artifact content to be parsed
+   * @param artifactName
+   *          name of the artifact
+   * @return a list of model artifacts
+   */
+  public List<Artifact> parse(byte[] artifactPayload, String artifactName) {
+    String payload = new String(artifactPayload);
+    List<Artifact> modelList = new ArrayList<Artifact>();
+
+    try {
+      // Artifact could be JSON or XML
+      if (JsonXmlConverter.isValidJson(payload)) {
+        payload = JsonXmlConverter.convertJsonToXml(payload);
+      }
+
+      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+      DocumentBuilder builder = factory.newDocumentBuilder();
+      InputSource is = new InputSource(new StringReader(payload));
+      Document doc = builder.parse(is);
+
+      if ((doc.getDocumentElement().getNodeName().equalsIgnoreCase(MODEL_ELEMENT))
+          || (doc.getDocumentElement().getNodeName().equalsIgnoreCase(NAMED_QUERY_ELEMENT))) {
+        ModelArtifact model = parseModel(doc.getDocumentElement(), payload);
+        if (model != null) {
+          modelList.add(model);
+        } else {
+          // TODO: A WARN message?
+          logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT,
+              "Unable to parse artifact " + artifactName);
+        }
+      } else if ((doc.getDocumentElement().getNodeName().equalsIgnoreCase(MODELS_ELEMENT))
+          || (doc.getDocumentElement().getNodeName().equalsIgnoreCase(NAMED_QUERIES_ELEMENT))) {
+        // The complete set of models/named-queries were contained in this
+        // artifact
+        NodeList nodeList = doc.getDocumentElement().getChildNodes();
+        for (int i = 0; i < nodeList.getLength(); i++) {
+          Node childNode = nodeList.item(i);
+          if ((childNode.getNodeName().equalsIgnoreCase(MODEL_ELEMENT))
+              || (childNode.getNodeName().equalsIgnoreCase(NAMED_QUERY_ELEMENT))) {
+            String modelPayload = nodeToString(childNode);
+            ModelArtifact model = parseModel(childNode, modelPayload);
+            if (model != null) {
+              modelList.add(model);
+            } else {
+              // TODO: A WARN message?
+              logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT,
+                  "Unable to parse artifact " + artifactName);
+              modelList.clear();
+              break;
+            }
+          }
+        }
+      }
+    } catch (Exception ex) {
+      // This may not be an error. We may be receiving an artifact that is
+      // unrelated
+      // to models. In this case, we just ignore it.
+      // TODO: A WARN message?
+      logger.info(ModelLoaderMsgs.DISTRIBUTION_EVENT,
+          "Unable to parse artifact " + artifactName + ": " + ex.getLocalizedMessage());
+    }
+
+    return modelList;
+  }
+
+  private ModelArtifact parseModel(Node modelNode, String payload) {
+    ModelArtifact model = new ModelArtifact();
+    model.setPayload(payload);
+
+    if (modelNode.getNodeName().equalsIgnoreCase(MODEL_ELEMENT)) {
+      model.setType(ArtifactType.MODEL);
+    } else {
+      model.setType(ArtifactType.NAMED_QUERY);
+    }
+
+    parseNode(modelNode, model);
+
+    if (model.getNameVersionId() == null) {
+      return null;
+    }
+
+    return model;
+  }
+
+  private void parseNode(Node node, ModelArtifact model) {
+    if (node.getNodeName().equalsIgnoreCase(MODEL_NAME_VERSION_ID)) {
+      model.setNameVersionId(node.getTextContent().trim());
+    } else if (node.getNodeName().equalsIgnoreCase(NAMED_QUERY_VERSION_ID)) {
+      model.setNameVersionId(node.getTextContent().trim());
+    } else if (node.getNodeName().equalsIgnoreCase(RELATIONSHIP_DATA)) {
+      parseRelationshipNode(node, model);
+    } else {
+      NodeList nodeList = node.getChildNodes();
+      for (int i = 0; i < nodeList.getLength(); i++) {
+        Node childNode = nodeList.item(i);
+        parseNode(childNode, model);
+      }
+    }
+  }
+
+  private void parseRelationshipNode(Node node, ModelArtifact model) {
+    String key = null;
+    String value = null;
+
+    NodeList nodeList = node.getChildNodes();
+    for (int i = 0; i < nodeList.getLength(); i++) {
+      Node childNode = nodeList.item(i);
+      if (childNode.getNodeName().equalsIgnoreCase(RELATIONSHIP_KEY)) {
+        key = childNode.getTextContent().trim();
+      } else if (childNode.getNodeName().equalsIgnoreCase(RELATIONSHIP_VALUE)) {
+        value = childNode.getTextContent().trim();
+      }
+    }
+
+    if ((key != null) && (key.equalsIgnoreCase(MODEL_ELEMENT_RELATIONSHIP_KEY))) {
+      if (value != null) {
+        model.addDependentModelId(value);
+      }
+    }
+  }
+
+  private String nodeToString(Node node) throws TransformerException {
+    StringWriter sw = new StringWriter();
+    Transformer transfomer = TransformerFactory.newInstance().newTransformer();
+    transfomer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
+    transfomer.transform(new DOMSource(node), new StreamResult(sw));
+    return sw.toString();
+  }
+}
diff --git a/src/main/java/org/openecomp/modelloader/entity/model/ModelSorter.java b/src/main/java/org/openecomp/modelloader/entity/model/ModelSorter.java
new file mode 100644
index 0000000..4dcda71
--- /dev/null
+++ b/src/main/java/org/openecomp/modelloader/entity/model/ModelSorter.java
@@ -0,0 +1,233 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * MODEL LOADER SERVICE
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.openecomp.modelloader.entity.model;
+
+import jline.internal.Log;
+
+import org.openecomp.modelloader.entity.Artifact;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Utility class to sort the given Models according to their dependencies.
+ * Example: Given a list of Models [A, B, C] where B depends on A, and A depends
+ * on C, the sorted result will be [C, A, B]
+ */
+public class ModelSorter {
+
+  /**
+   * Wraps a Model object to form dependencies other Models using Edges.
+   */
+  static class Node {
+    private final ModelArtifact model;
+    private final HashSet<Edge> inEdges;
+    private final HashSet<Edge> outEdges;
+
+    public Node(ModelArtifact model) {
+      this.model = model;
+      inEdges = new HashSet<Edge>();
+      outEdges = new HashSet<Edge>();
+    }
+
+    public Node addEdge(Node node) {
+      Edge edge = new Edge(this, node);
+      outEdges.add(edge);
+      node.inEdges.add(edge);
+      return this;
+    }
+
+    @Override
+    public String toString() {
+      return model.getNameVersionId();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      ModelArtifact otherModel = ((Node) other).model;
+      return this.model.getNameVersionId().equals(otherModel.getNameVersionId());
+    }
+
+    @Override
+    public int hashCode() {
+      return this.model.getNameVersionId().hashCode();
+
+    }
+  }
+
+  /**
+   * Represents a dependency between two Nodes.
+   */
+  static class Edge {
+    public final Node from;
+    public final Node to;
+
+    public Edge(Node from, Node to) {
+      this.from = from;
+      this.to = to;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      Edge edge = (Edge) obj;
+      return edge.from == from && edge.to == to;
+    }
+  }
+
+  /**
+   * Returns the list of models sorted by order of dependency.
+   * 
+   * @param originalList
+   *          the list that needs to be sorted
+   * @return a list of sorted models
+   */
+  public List<Artifact> sort(List<Artifact> originalList) {
+
+    if (originalList.size() <= 1) {
+      return originalList;
+    }
+
+    Collection<Node> nodes = createNodes(originalList);
+    Collection<Node> sortedNodes = sortNodes(nodes);
+
+    List<Artifact> sortedModelsList = new ArrayList<Artifact>(sortedNodes.size());
+    for (Node node : sortedNodes) {
+      sortedModelsList.add(node.model);
+    }
+
+    return sortedModelsList;
+  }
+
+  /**
+   * Create nodes from the list of models and their dependencies.
+   * 
+   * @param models
+   *          what the nodes creation is based upon
+   * @return Collection of Node objects
+   */
+  private Collection<Node> createNodes(Collection<Artifact> models) {
+
+    // load list of models into a map, so we can later replace referenceIds with
+    // real Models
+    HashMap<String, ModelArtifact> versionIdToModelMap = new HashMap<String, ModelArtifact>();
+    for (Artifact art : models) {
+      ModelArtifact ma = (ModelArtifact) art;
+      versionIdToModelMap.put(ma.getNameVersionId(), ma);
+    }
+
+    HashMap<String, Node> nodes = new HashMap<String, Node>();
+    // create a node for each model and its referenced models
+    for (Artifact art : models) {
+      ModelArtifact model = (ModelArtifact) art;
+
+      // node might have been created by another model referencing it
+      Node node = nodes.get(model.getNameVersionId());
+
+      if (null == node) {
+        node = new Node(model);
+        nodes.put(model.getNameVersionId(), node);
+      }
+
+      for (String referencedModelId : model.getDependentModelIds()) {
+        // node might have been created by another model referencing it
+        Node referencedNode = nodes.get(referencedModelId);
+
+        if (null == referencedNode) {
+          // create node
+          ModelArtifact referencedModel = versionIdToModelMap.get(referencedModelId);
+          if (referencedModel == null) {
+            Log.debug("ignoring " + referencedModelId);
+            continue; // referenced model not supplied, no need to sort it
+          }
+          referencedNode = new Node(referencedModel);
+          nodes.put(referencedModelId, referencedNode);
+        }
+        referencedNode.addEdge(node);
+      }
+    }
+
+    return nodes.values();
+  }
+
+  /**
+   * Sorts the given Nodes by order of dependency.
+   * 
+   * @param originalList
+   *          the collection of nodes to be sorted
+   * @return a sorted collection of the given nodes
+   */
+  private Collection<Node> sortNodes(Collection<Node> unsortedNodes) {
+
+    // L <- Empty list that will contain the sorted elements
+    ArrayList<Node> nodeList = new ArrayList<Node>();
+
+    // S <- Set of all nodes with no incoming edges
+    HashSet<Node> nodeSet = new HashSet<Node>();
+    for (Node unsortedNode : unsortedNodes) {
+      if (unsortedNode.inEdges.size() == 0) {
+        nodeSet.add(unsortedNode);
+      }
+    }
+
+    // while S is non-empty do
+    while (!nodeSet.isEmpty()) {
+      // remove a node n from S
+      Node node = nodeSet.iterator().next();
+      nodeSet.remove(node);
+
+      // insert n into L
+      nodeList.add(node);
+
+      // for each node m with an edge e from n to m do
+      for (Iterator<Edge> it = node.outEdges.iterator(); it.hasNext();) {
+        // remove edge e from the graph
+        Edge edge = it.next();
+        Node to = edge.to;
+        it.remove();// Remove edge from n
+        to.inEdges.remove(edge);// Remove edge from m
+
+        // if m has no other incoming edges then insert m into S
+        if (to.inEdges.isEmpty()) {
+          nodeSet.add(to);
+        }
+      }
+    }
+    // Check to see if all edges are removed
+    boolean cycle = false;
+    for (Node node : unsortedNodes) {
+      if (!node.inEdges.isEmpty()) {
+        cycle = true;
+        break;
+      }
+    }
+    if (cycle) {
+      throw new RuntimeException(
+          "Circular dependency present between models, topological sort not possible");
+    }
+
+    return nodeList;
+  }
+
+}