Process multi-OXM files

Gizmo service uses common OXM consumer to process multi-OXM files

Issue-ID: AAI-1196

Change-Id: If6dd50413a914984948272cd6bd98e2eeeaecedb
Signed-off-by: Sotiropoulos, Ioannis (is948x) <Ioannis.Sotiropoulos@amdocs.com>

5 files changed, 382 insertions, 23 deletions

diff --git a/src/main/java/org/onap/schema/OxmModelConfigTranslator.java b/src/main/java/org/onap/schema/OxmModelConfigTranslator.java
new file mode 100644
index 0000000..f6b9e70
--- /dev/null
+++ b/src/main/java/org/onap/schema/OxmModelConfigTranslator.java
@@ -0,0 +1,100 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.schema;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.ServiceConfigurationError;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.onap.aai.setup.ConfigTranslator;
+import org.onap.aai.setup.SchemaLocationsBean;
+import org.onap.aai.setup.Version;
+
+public class OxmModelConfigTranslator extends ConfigTranslator {
+
+    public OxmModelConfigTranslator(SchemaLocationsBean bean) {
+        super(bean);
+    }
+
+    @Override
+    public Map<Version, List<String>> getNodeFiles() {
+        String nodeDirectory = bean.getNodeDirectory();
+        if (nodeDirectory == null) {
+            throw new ServiceConfigurationError(
+                    "Node(s) directory is empty in the schema location bean (" + bean.getSchemaConfigLocation() + ")");
+        }
+        try {
+            return getVersionMap(Paths.get(nodeDirectory), "*_v*.xml");
+        } catch (IOException e) {
+            throw new ServiceConfigurationError("Failed to read node(s) directory " + getPath(nodeDirectory), e);
+        }
+    }
+
+    @Override
+    public Map<Version, List<String>> getEdgeFiles() {
+        String edgeDirectory = bean.getEdgeDirectory();
+        if (edgeDirectory == null) {
+            throw new ServiceConfigurationError(
+                    "Edge(s) directory is empty in the schema location bean (" + bean.getSchemaConfigLocation() + ")");
+        }
+        try {
+            return getVersionMap(Paths.get(edgeDirectory), "*_v*.json");
+        } catch (IOException e) {
+            throw new ServiceConfigurationError("Failed to read edge(s) directory " + getPath(edgeDirectory), e);
+        }
+    }
+
+    private String getPath(String nodeDirectory) {
+        return Paths.get(nodeDirectory).toAbsolutePath().toString();
+    }
+
+    /**
+     * Creates a map containing each OXM Version and the matching OXM file path(s)
+     *
+     * @param folderPath the folder/directory containing the OXM files
+     * @param fileSuffix
+     * @return a new Map object (may be empty)
+     * @throws IOException if there is a problem reading the specified directory path
+     */
+    private Map<Version, List<String>> getVersionMap(Path folderPath, String globPattern) throws IOException {
+        final PathMatcher filter = folderPath.getFileSystem().getPathMatcher("glob:**/" + globPattern);
+        try (final Stream<Path> stream = Files.list(folderPath)) {
+            return stream.filter(filter::matches).map(Path::toString).filter(p -> getVersionFromPath(p) != null)
+                    .collect(Collectors.groupingBy(this::getVersionFromPath));
+        }
+    }
+
+    private Version getVersionFromPath(String pathName) {
+        String version = "V" + pathName.replaceAll("^.*\\/", "").replaceAll("\\D+", "");
+        try {
+            return Version.valueOf(version);
+        } catch (IllegalArgumentException e) {
+            return null;
+        }
+    }
+
+}
diff --git a/src/main/java/org/onap/schema/OxmModelLoader.java b/src/main/java/org/onap/schema/OxmModelLoader.java
new file mode 100644
index 0000000..869df8f
--- /dev/null
+++ b/src/main/java/org/onap/schema/OxmModelLoader.java
@@ -0,0 +1,161 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.schema;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.ws.rs.core.Response.Status;
+import org.eclipse.persistence.jaxb.dynamic.DynamicJAXBContext;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.aai.nodes.NodeIngestor;
+import org.onap.aai.setup.ConfigTranslator;
+import org.onap.aai.setup.SchemaLocationsBean;
+import org.onap.aai.setup.Version;
+import org.onap.crud.exception.CrudException;
+import org.onap.crud.logging.CrudServiceMsgs;
+import org.onap.schema.util.SchemaIngestPropertyReader;
+
+public class OxmModelLoader {
+
+    private static Map<String, DynamicJAXBContext> versionContextMap =
+            new ConcurrentHashMap<>();
+
+    static final Pattern p = Pattern.compile("aai_oxm_(.*).xml");
+    static final Pattern versionPattern = Pattern.compile("V(\\d*)");
+
+    private static org.onap.aai.cl.api.Logger logger =
+            LoggerFactory.getInstance().getLogger(OxmModelLoader.class.getName());
+
+    private OxmModelLoader() {
+    }
+
+    /**
+     * Finds all OXM model files
+     *
+     * @throws SpikeException
+     * @throws IOException
+     *
+     */
+    public static synchronized void loadModels() throws CrudException {
+        SchemaIngestPropertyReader schemaIngestPropertyReader = new SchemaIngestPropertyReader();
+
+        SchemaLocationsBean schemaLocationsBean = new SchemaLocationsBean();
+        schemaLocationsBean.setNodeDirectory(schemaIngestPropertyReader.getNodeDir());
+        ConfigTranslator configTranslator = new OxmModelConfigTranslator(schemaLocationsBean);
+        NodeIngestor nodeIngestor = new NodeIngestor(configTranslator);
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("Loading OXM Models");
+        }
+
+        for (Version oxmVersion : Version.values()) {
+            DynamicJAXBContext jaxbContext = nodeIngestor.getContextForVersion(oxmVersion);
+            if (jaxbContext != null) {
+                loadModel(oxmVersion.toString().toLowerCase(), jaxbContext);
+            }
+        }
+    }
+
+
+    private static synchronized void loadModel(String oxmVersion, DynamicJAXBContext jaxbContext) {
+        versionContextMap.put(oxmVersion, jaxbContext);
+        logger.info(CrudServiceMsgs.LOADED_OXM_FILE, oxmVersion);
+    }
+
+    /**
+     * Retrieves the JAXB context for the specified OXM model version.
+     *
+     * @param version - The OXM version that we want the JAXB context for.
+     *
+     * @return - A JAXB context derived from the OXM model schema.
+     *
+     * @throws SpikeException
+     */
+    public static DynamicJAXBContext getContextForVersion(String version) throws CrudException {
+
+        // If we haven't already loaded in the available OXM models, then do so now.
+        if (versionContextMap == null || versionContextMap.isEmpty()) {
+            loadModels();
+        } else if (!versionContextMap.containsKey(version)) {
+			logger.error(CrudServiceMsgs.OXM_LOAD_ERROR, "Error loading oxm model: " + version);
+            throw new CrudException("Error loading oxm model: " + version, Status.INTERNAL_SERVER_ERROR);
+        }
+
+        return versionContextMap.get(version);
+    }
+
+    public static String getLatestVersion() throws CrudException {
+
+        // If we haven't already loaded in the available OXM models, then do so now.
+        if (versionContextMap == null || versionContextMap.isEmpty()) {
+            loadModels();
+        }
+
+        // If there are still no models available, then there's not much we can do...
+        if (versionContextMap.isEmpty()) {
+			logger.error(CrudServiceMsgs.OXM_LOAD_ERROR, "No available OXM schemas to get latest version for.");
+            throw new CrudException("No available OXM schemas to get latest version for.",
+                    Status.INTERNAL_SERVER_ERROR);
+        }
+
+        // Iterate over the available model versions to determine which is the most
+        // recent.
+        Integer latestVersion = null;
+        String latestVersionStr = null;
+        for (String versionKey : versionContextMap.keySet()) {
+
+            Matcher matcher = versionPattern.matcher(versionKey.toUpperCase());
+            if (matcher.find()) {
+
+                int currentVersion = Integer.parseInt(matcher.group(1));
+
+                if ((latestVersion == null) || (currentVersion > latestVersion)) {
+                    latestVersion = currentVersion;
+                    latestVersionStr = versionKey;
+                }
+            }
+        }
+
+        return latestVersionStr;
+    }
+
+    /**
+     * Retrieves the map of all JAXB context objects that have been created by importing the
+     * available OXM model schemas.
+     *
+     * @return - Map of JAXB context objects.
+     */
+    public static Map<String, DynamicJAXBContext> getVersionContextMap() {
+        return versionContextMap;
+    }
+
+    /**
+     * Assigns the map of all JAXB context objects.
+     *
+     * @param versionContextMap
+     */
+    public static void setVersionContextMap(Map<String, DynamicJAXBContext> versionContextMap) {
+        OxmModelLoader.versionContextMap = versionContextMap;
+    }
+}
diff --git a/src/main/java/org/onap/schema/OxmModelValidator.java b/src/main/java/org/onap/schema/OxmModelValidator.java
index 6bc8bcf..24d4a8c 100644
--- a/src/main/java/org/onap/schema/OxmModelValidator.java
+++ b/src/main/java/org/onap/schema/OxmModelValidator.java
@@ -20,27 +20,29 @@
  */
 package org.onap.schema;
 
-import com.google.common.base.CaseFormat;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonNull;
-
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import javax.ws.rs.core.Response.Status;
 import org.eclipse.persistence.dynamic.DynamicType;
 import org.eclipse.persistence.internal.helper.DatabaseField;
 import org.eclipse.persistence.jaxb.dynamic.DynamicJAXBContext;
 import org.eclipse.persistence.mappings.DatabaseMapping;
 import org.eclipse.persistence.oxm.XMLField;
-import org.onap.aaiutils.oxm.OxmModelLoader;
+import org.onap.aai.cl.api.Logger;
+import org.onap.aai.cl.eelf.LoggerFactory;
 import org.onap.crud.entity.Vertex;
 import org.onap.crud.exception.CrudException;
+import org.onap.crud.logging.CrudServiceMsgs;
 import org.onap.crud.util.CrudServiceConstants;
 import org.onap.crud.util.CrudServiceUtil;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import javax.ws.rs.core.Response.Status;
+import com.google.common.base.CaseFormat;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
 
 public class OxmModelValidator {
+  private static Logger logger = LoggerFactory.getInstance().getLogger(OxmModelValidator.class.getName());
+	
   public enum Metadata {
     NODE_TYPE("aai-node-type"),
     URI("aai-uri"),
@@ -81,7 +83,8 @@ public class OxmModelValidator {
 
     Map<String, Object> result = new HashMap<String, Object>();
     if (jaxbContext == null) {
-      throw new CrudException("", Status.NOT_FOUND);
+      logger.error(CrudServiceMsgs.OXM_LOAD_ERROR, "Error loading oxm model: " + version);
+      throw new CrudException("Error loading oxm model: " + version, Status.NOT_FOUND);
     }
     final DynamicType modelObjectType = jaxbContext.getDynamicType(
         CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_CAMEL, type)));
@@ -120,12 +123,15 @@ public class OxmModelValidator {
     DynamicJAXBContext jaxbContext = null;
     try {
       jaxbContext = OxmModelLoader.getContextForVersion(version);
+    } catch (CrudException ce) {
+      throw new CrudException(ce.getMessage(), ce.getHttpStatus());
     } catch (Exception e) {
       throw new CrudException(e);
     }
 
     if (jaxbContext == null) {
-      throw new CrudException("", Status.NOT_FOUND);
+      logger.error(CrudServiceMsgs.OXM_LOAD_ERROR, "Error loading oxm model: " + version);
+      throw new CrudException("Error loading oxm model: " + version, Status.NOT_FOUND);
     }
     // Determine if the Object part is a collection type in the model
     // definition
@@ -133,7 +139,8 @@ public class OxmModelValidator {
         CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, CaseFormat.LOWER_HYPHEN.to(CaseFormat.UPPER_CAMEL, type)));
 
     if (modelObjectType == null) {
-      throw new CrudException("", Status.NOT_FOUND);
+      logger.error(CrudServiceMsgs.INVALID_OXM_FILE, "Object of collection type not found: " + type);
+      throw new CrudException("Object of collection type not found: " + type, Status.NOT_FOUND);
     }
 
     if (modelObjectType.getDescriptor().getMappings().size() == 1
@@ -142,9 +149,9 @@ public class OxmModelValidator {
       childJavaObjectName = CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, childJavaObjectName);
       final DynamicType childObjectType = jaxbContext.getDynamicType(childJavaObjectName);
       if (childObjectType == null) {
-        // Should not happen as child object is defined in oxm model
-        // itself
-        throw new CrudException("", Status.NOT_FOUND);
+        // Should not happen as child object is defined in oxm model itself
+        logger.error(CrudServiceMsgs.INVALID_OXM_FILE, "Child Object Type for Java Object not found: " + childJavaObjectName);
+        throw new CrudException("Child Object Type for Java Object not found: " + childJavaObjectName, Status.NOT_FOUND);
       }
       return childObjectType.getDescriptor().getTableName();
     } else {
@@ -231,6 +238,8 @@ public class OxmModelValidator {
       }
 
       return modelVertexBuilder.build();
+    } catch (CrudException ce) {
+      throw new CrudException(ce.getMessage(), ce.getHttpStatus());
     } catch (Exception e) {
       throw new CrudException(e.getMessage(), Status.BAD_REQUEST);
     }
diff --git a/src/main/java/org/onap/schema/RelationshipSchemaLoader.java b/src/main/java/org/onap/schema/RelationshipSchemaLoader.java
index 0e10fc3..9a9a37c 100644
--- a/src/main/java/org/onap/schema/RelationshipSchemaLoader.java
+++ b/src/main/java/org/onap/schema/RelationshipSchemaLoader.java
@@ -41,9 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-
 import javax.ws.rs.core.Response.Status;
-
 import org.apache.commons.io.IOUtils;
 import org.onap.aai.cl.eelf.LoggerFactory;
 import org.onap.crud.exception.CrudException;
@@ -125,8 +123,8 @@ public class RelationshipSchemaLoader {
         rulesFiles.add(file);
         existingFiles.add(filename(file));
       }
-      
-      // Get DBEdgeRules from the jar on the classpath.  Don't include any that conflict with files which 
+
+      // Get DBEdgeRules from the jar on the classpath.  Don't include any that conflict with files which
       // were dropped manually.
       Resource[] rawResourceList = rulesResolver.getResources("classpath*:/dbedgerules/DbEdgeRules*" + fileExt);
       List<Resource> prunedResourceList = new ArrayList<Resource>();
@@ -135,10 +133,10 @@ public class RelationshipSchemaLoader {
           prunedResourceList.add(resource);
         }
       }
-      
+
       rulesFiles.addAll(Arrays.stream(prunedResourceList.toArray(new Resource[prunedResourceList.size()]))
           .filter(r -> !myMatcher(rulesPattern, r.getFilename()).isEmpty()).collect(Collectors.toList()));
-      
+
       // This gets all the objects of type "File" from external directory (not
       // on the classpath)
       // 1. From an external directory (one not on the classpath) we get all the
@@ -271,4 +269,4 @@ public class RelationshipSchemaLoader {
     Matcher m = p.matcher(s);
     return m.matches() ? m.group(1) : "";
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/org/onap/schema/util/SchemaIngestPropertyReader.java b/src/main/java/org/onap/schema/util/SchemaIngestPropertyReader.java
new file mode 100644
index 0000000..71ea47e
--- /dev/null
+++ b/src/main/java/org/onap/schema/util/SchemaIngestPropertyReader.java
@@ -0,0 +1,91 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.schema.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Properties;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.crud.exception.CrudException;
+import org.onap.crud.logging.CrudServiceMsgs;
+
+public class SchemaIngestPropertyReader {
+
+    private static final String SCHEMA_INGEST_PROPERTIES_FILE = "schemaIngest.properties";
+
+    private static final String SCHEMA_INGEST_PROPERTIES_LOCATION =
+            System.getProperty("CONFIG_HOME") + "/" + SCHEMA_INGEST_PROPERTIES_FILE;
+
+    private static org.onap.aai.cl.api.Logger logger =
+            LoggerFactory.getInstance().getLogger(SchemaIngestPropertyReader.class.getName());
+
+    /**
+     * Gets the location of the OXM
+     *
+     * @return
+     * @throws SpikeException
+     * @throws IOException
+     */
+    public String getNodeDir() throws CrudException {
+
+        Properties prop = new Properties();
+        try {
+            prop = loadFromFile(SCHEMA_INGEST_PROPERTIES_LOCATION);
+        } catch (NoSuchFileException e) {
+            // if file not found, try via classpath
+            try {
+                prop = loadFromClasspath(SCHEMA_INGEST_PROPERTIES_FILE);
+            } catch (URISyntaxException | IOException e1) {
+                logger.error(CrudServiceMsgs.SCHEMA_INGEST_LOAD_ERROR, e1.getMessage());
+                throw new CrudException("Failed to load schemaIngest.properties", e1);
+            }
+        } catch (IOException e) {
+            logger.error(CrudServiceMsgs.SCHEMA_INGEST_LOAD_ERROR, e.getMessage());
+            throw new CrudException("Failed to load schemaIngest.properties", e);
+        }
+        return prop.getProperty("nodeDir");
+    }
+
+    private Properties loadFromFile(String filename) throws IOException {
+        Path configLocation = Paths.get(filename);
+        try (InputStream stream = Files.newInputStream(configLocation)) {
+            return loadProperties(stream);
+        }
+    }
+
+    private Properties loadFromClasspath(String resourceName) throws URISyntaxException, IOException {
+        Path path = Paths.get(ClassLoader.getSystemResource(resourceName).toURI());
+        try (InputStream stream = Files.newInputStream(path)) {
+            return loadProperties(stream);
+        }
+    }
+
+    private Properties loadProperties(InputStream stream) throws IOException {
+        Properties config = new Properties();
+        config.load(stream);
+        return config;
+    }
+}
\ No newline at end of file