Updating Search service to be ES 6.1.2 compliant

Payload to Elastic Search is translated to comply ES6.1.2
PUT and POST calls now have content-type header
Added functionality for dynamic templates

Change-Id: I2a44a8a9999ec01a3bad1fb6999fe35bb6ef70d1
Issue-ID: AAI-1376
Signed-off-by: Edwin Lawrance <Edwin.Lawrance@amdocs.com>

3 files changed, 129 insertions, 7 deletions

diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
index 626ea8c..90adbd7 100644
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
@@ -48,6 +48,7 @@ import org.onap.aai.sa.searchdbabstraction.entity.SearchOperationResult;
 import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
 import org.onap.aai.sa.searchdbabstraction.util.AggregationParsingUtil;
 import org.onap.aai.sa.searchdbabstraction.util.DocumentSchemaUtil;
+import org.onap.aai.sa.searchdbabstraction.util.ElasticSearchPayloadTranslator;
 import org.onap.aai.sa.searchdbabstraction.util.SearchDbConstants;
 import org.onap.aai.cl.api.LogFields;
 import org.onap.aai.cl.api.LogLine;
@@ -179,7 +180,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
         //result.setResult("{\"index\": \"" + index + ", \"type\": \"" + DEFAULT_TYPE + "\"}");
       }
 
-    } catch (DocumentStoreOperationException e) {
+    } catch (DocumentStoreOperationException | IOException e) {
 
       result.setFailureCause("Document store operation failure.  Cause: " + e.getMessage());
     }
@@ -350,6 +351,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
     try {
       conn.setRequestMethod("PUT");
+      conn.setRequestProperty("Content-Type", "application/json");
     } catch (ProtocolException e) {
       shutdownConnection(conn);
       throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
@@ -365,7 +367,11 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
     sb.append(indexMappings);
     sb.append("}}");
 
-    attachContent(conn, sb.toString());
+    try {
+    	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString()));
+    } catch(IOException e) {
+    	throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e);
+    }
 
     logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL());
     logger.debug("Request content: " + sb.toString());
@@ -407,12 +413,17 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
     try {
       conn.setRequestMethod("PUT");
+      conn.setRequestProperty("Content-Type", "application/json");
     } catch (ProtocolException e) {
       shutdownConnection(conn);
       throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
     }
 
-    attachContent(conn, settingsAndMappings);
+    try {
+    	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings));
+    } catch(IOException e) {
+    	throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e);
+    }
     handleResponse(conn, result);
 
     // Generate a metrics log so we can track how long the operation took.
@@ -557,10 +568,15 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
   private void attachDocument(HttpURLConnection conn, DocumentStoreDataEntity doc)
       throws DocumentStoreOperationException {
-    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+//    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
+    conn.setRequestProperty("Content-Type", "application/json");
     conn.setRequestProperty("Connection", "Close");
-
-    attachContent(conn, doc.getContentInJson());
+    
+    try {
+    	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson()));
+    } catch(IOException e) {
+    	throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e);
+    }
   }
 
   private DocumentOperationResult checkDocumentExistence(String indexName,
@@ -650,6 +666,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
     try {
       conn.setRequestMethod("PUT");
+      conn.setRequestProperty("Content-Type", "application/json");
     } catch (ProtocolException e) {
       shutdownConnection(conn);
       throw new DocumentStoreOperationException("Failed to set HTTP request method to PUT.", e);
@@ -823,6 +840,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
     try {
       conn.setRequestMethod("POST");
+      conn.setRequestProperty("Content-Type", "application/json");
     } catch (ProtocolException e) {
       shutdownConnection(conn);
       throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e);
@@ -872,6 +890,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 
     try {
       conn.setRequestMethod("POST");
+      conn.setRequestProperty("Content-Type", "application/json");
     } catch (ProtocolException e) {
       shutdownConnection(conn);
       throw new DocumentStoreOperationException("Failed to set HTTP request method to POST.", e);
diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java
index 8c79022..3dc03f9 100644
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/DocumentSchemaUtil.java
@@ -23,15 +23,23 @@ package org.onap.aai.sa.searchdbabstraction.util;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.commons.io.IOUtils;
 import org.onap.aai.sa.rest.DocumentFieldSchema;
 import org.onap.aai.sa.rest.DocumentSchema;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 public class DocumentSchemaUtil {
 
+  private static String dynamicCustomMapping = null;
+  private static final String DYNAMIC_CUSTOM_TEMPALTE_FILE = System.getProperty("CONFIG_HOME") + File.separator 
+		  + "dynamic-custom-template.json";
+  
   public static String generateDocumentMappings(String documentSchema)
       throws JsonParseException, JsonMappingException, IOException {
 
@@ -42,11 +50,23 @@ public class DocumentSchemaUtil {
     return generateDocumentMappings(schema);
   }
 
-  public static String generateDocumentMappings(DocumentSchema schema) {
+  public static String generateDocumentMappings(DocumentSchema schema) throws IOException {
+	  
+	// Adding dynamic template to add fielddata=true to dynamic fields of type "string"  
+	// in order to avoid aggregation queries breaking in ESv6.1.2
+	if(dynamicCustomMapping == null) {
+		try {
+			dynamicCustomMapping = IOUtils.toString(new FileInputStream(DYNAMIC_CUSTOM_TEMPALTE_FILE), "UTF-8").replaceAll("\\s+", "");
+		} catch (IOException e) {
+			throw new IOException("Dynamic Custom template configuration went wrong! Please check for the correct template file.", e);
+		}
+	}
 
     // Now, generate the Elastic Search mapping json and return it.
     StringBuilder sb = new StringBuilder();
     sb.append("{");
+    // Adding custom mapping which adds fielddata=true to dynamic fields of type "string"
+    sb.append(dynamicCustomMapping != null ? dynamicCustomMapping : "");
     sb.append("\"properties\": {");
 
     generateFieldMappings(schema.getFields(), sb);
diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
new file mode 100644
index 0000000..2e97103
--- /dev/null
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
@@ -0,0 +1,83 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017-2018 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.sa.searchdbabstraction.util;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.onap.aai.cl.api.Logger;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
+
+
+/**
+ * This class as the name suggests is to translate the payload of PUT & POST requests
+ * to ElasticSearch (ES) to its compatible syntax, specially compatible with ES v6 or above.
+ * 
+ * For example, data type such as "string" is now replaced by "text" or "keyword". 
+ * 
+ * So this class will make those translations reading off from a json configuration file, therefore
+ * the configuration can be updated with new translations as and when required without touching the code. 
+ * 
+ * @author EDWINL
+ *
+ */
+public class ElasticSearchPayloadTranslator {
+
+	private static Logger logger = LoggerFactory.getInstance().getLogger(ElasticSearchPayloadTranslator.class.getName());
+	private static final String CONFIG_DIRECTORY = System.getProperty("CONFIG_HOME");
+	private static final String ES_PAYLOAD_TRANSLATION_FILE = "es-payload-translation.json";
+
+
+	/**
+	 *  Using String replacement to translate the payload to ES compatible version
+	 * 
+	 * @param source
+	 * @return translated payload in String
+	 * @throws IOException
+	 */
+	public static String translateESPayload(String source) throws IOException {
+		logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]");
+		String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE;
+		
+		String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces
+		JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
+				new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
+
+		JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+
+		for(Object obj : attrTranslations) {
+			JSONObject jsonObj = ((JSONObject)obj);
+			String from = jsonObj.get("from").toString();
+			String to = jsonObj.get("to").toString();
+			if(translatedPayload.indexOf(from) > 0) {
+				translatedPayload = translatedPayload.replaceAll(from, to);
+			}
+		}
+
+		logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload);
+		return translatedPayload;
+	}
+}