Fix to not remove whitespaces in the payload to ES

Change-Id: I3547e79ff27220a49539d9cfdf48ac534f155f10
Issue-ID: AAI-1596
Signed-off-by: Edwin Lawrance <Edwin.Lawrance@amdocs.com>

6 files changed, 87 insertions, 47 deletions

diff --git a/pom.xml b/pom.xml
index 3a3a67e..e6768cd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -69,7 +69,6 @@ limitations under the License.
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path</artifactId>
             <version>2.2.0</version>
-            <scope>test</scope>
         </dependency>
 
         <dependency>
diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
index 90adbd7..98a254c 100644
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/elasticsearch/dao/ElasticSearchHttpController.java
@@ -370,7 +370,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
     try {
     	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(sb.toString()));
     } catch(IOException e) {
-    	throw new DocumentStoreOperationException("Error in translating Index payload to make it ES compliant.", e);
+    	logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e);
+    	throw new DocumentStoreOperationException(e.getMessage(), e);
     }
 
     logger.debug("\ncreateTable(), Sending 'PUT' request to URL : " + conn.getURL());
@@ -422,7 +423,8 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
     try {
     	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(settingsAndMappings));
     } catch(IOException e) {
-    	throw new DocumentStoreOperationException("Error in translating DynamicIndex payload to make it ES compliant.", e);
+    	logger.error(SearchDbMsgs.INDEX_CREATE_FAILURE, e);
+    	throw new DocumentStoreOperationException(e.getMessage());
     }
     handleResponse(conn, result);
 
@@ -571,12 +573,7 @@ public class ElasticSearchHttpController implements DocumentStoreInterface {
 //    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
     conn.setRequestProperty("Content-Type", "application/json");
     conn.setRequestProperty("Connection", "Close");
-    
-    try {
-    	attachContent(conn, ElasticSearchPayloadTranslator.translateESPayload(doc.getContentInJson()));
-    } catch(IOException e) {
-    	throw new DocumentStoreOperationException("Error in translating Document payload to make it ES compliant.", e);
-    }
+    attachContent(conn, doc.getContentInJson());
   }
 
   private DocumentOperationResult checkDocumentExistence(String indexName,
diff --git a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
index 2e97103..97f0ca1 100644
--- a/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
+++ b/src/main/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslator.java
@@ -23,14 +23,20 @@ package org.onap.aai.sa.searchdbabstraction.util;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.io.IOUtils;
 import org.json.JSONArray;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.onap.aai.cl.api.Logger;
 import org.onap.aai.cl.eelf.LoggerFactory;
 import org.onap.aai.sa.searchdbabstraction.logging.SearchDbMsgs;
 
+import com.jayway.jsonpath.DocumentContext;
+import com.jayway.jsonpath.JsonPath;
+
 
 /**
  * This class as the name suggests is to translate the payload of PUT & POST requests
@@ -52,8 +58,9 @@ public class ElasticSearchPayloadTranslator {
 
 
 	/**
-	 *  Using String replacement to translate the payload to ES compatible version
-	 * 
+	 *  Using JSON Path query to filter objects to translate the payload to ES compatible version
+	 *  The filter queries and the replacement attributes are configured in the es-payload-translation.json file.
+	 *  
 	 * @param source
 	 * @return translated payload in String
 	 * @throws IOException
@@ -61,23 +68,33 @@ public class ElasticSearchPayloadTranslator {
 	public static String translateESPayload(String source) throws IOException {
 		logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "translateESPayload, method-params[ source=" + source + "]");
 		String pathToTranslationFile = CONFIG_DIRECTORY + File.separator + ES_PAYLOAD_TRANSLATION_FILE;
-		
-		String translatedPayload = source.replaceAll("\\s+", ""); // removing whitespaces
-		JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
-				new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
 
-		JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+		try {
+			
+			JSONObject translationConfigPayload = new JSONObject(IOUtils.toString(
+					new FileInputStream(new File(pathToTranslationFile)), "UTF-8"));
+			JSONArray attrTranslations = translationConfigPayload.getJSONArray("attr-translations");
+			DocumentContext payloadToTranslate = JsonPath.parse(source);
 
-		for(Object obj : attrTranslations) {
-			JSONObject jsonObj = ((JSONObject)obj);
-			String from = jsonObj.get("from").toString();
-			String to = jsonObj.get("to").toString();
-			if(translatedPayload.indexOf(from) > 0) {
-				translatedPayload = translatedPayload.replaceAll(from, to);
+			for(Object obj : attrTranslations) {
+				JSONObject jsonObj = ((JSONObject) obj);
+				String query = jsonObj.get("query").toString();
+				JSONObject attrToUpdate = (JSONObject) jsonObj.get("update");
+				List<Map<String, Object>> filteredObjects = payloadToTranslate.read(query);
+				for(Map<String, Object> objMap : filteredObjects) {
+					objMap.putAll(attrToUpdate.toMap());
+				}
 			}
-		}
-
-		logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+translatedPayload);
-		return translatedPayload;
+			
+			logger.info(SearchDbMsgs.PROCESS_PAYLOAD_QUERY, "Payload after translation: "+payloadToTranslate.jsonString());
+			return payloadToTranslate.jsonString();
+			
+		} catch (JSONException | IOException e) {
+			logger.error(SearchDbMsgs.FILTERS_CONFIG_FAILURE, e, ES_PAYLOAD_TRANSLATION_FILE, e.getMessage());
+			if(e instanceof JSONException) {
+				throw new IOException("Payload translation configuration looks corrupted. Please correct!", e);
+			}
+			throw new IOException("Error in configuring payload translation file. Please check if it exists.", e);
+		}	
 	}
 }
diff --git a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
index 11b2acb..ba5cb48 100644
--- a/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
+++ b/src/test/java/org/onap/aai/sa/searchdbabstraction/util/ElasticSearchPayloadTranslatorTest.java
@@ -21,16 +21,19 @@
 package org.onap.aai.sa.searchdbabstraction.util;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 
 import java.io.File;
+import java.io.FileInputStream;
 
+import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Test;
 import org.onap.aai.sa.rest.TestUtils;
 
 public class ElasticSearchPayloadTranslatorTest {
 	
-	private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/simpleDocument.json";
+	private final String SIMPLE_DOC_SCHEMA_JSON = "src/test/resources/json/index-mapping.json";
 
 	@Before
 	public void setup() throws Exception {
@@ -38,13 +41,17 @@ public class ElasticSearchPayloadTranslatorTest {
 	}
 	
 	@Test
-	public void testPayloadTranslation_FromStringToText() throws Exception {
+	public void testPayloadTranslation() throws Exception {
+		String expectedErrMsg = "Sample error message for whitespace check";
 		File schemaFile = new File(SIMPLE_DOC_SCHEMA_JSON);
-	    String documentJson = TestUtils.readFileToString(schemaFile);
-	    assertTrue(documentJson.contains("\"data-type\":\"string\""));
-		assertTrue(documentJson.contains("\"searchable\":true"));
+	    String documentJson = IOUtils.toString(new FileInputStream(schemaFile), "UTF-8");
+	    assertTrue(documentJson.contains("\"type\": \"string\""));
+		assertTrue(documentJson.contains("\"index\": \"analyzed\""));
 		String translatedPayload = ElasticSearchPayloadTranslator.translateESPayload(documentJson);
-		assertTrue(translatedPayload.contains("\"data-type\":\"text\""));
+		assertTrue(translatedPayload.contains("\"type\":\"text\""));
 		assertTrue(translatedPayload.contains("\"index\":true"));
+		assertTrue(translatedPayload.contains("\"fielddata\":true"));
+		assertFalse(documentJson.contains("\"index\":\"analyzed\""));
+		assertTrue(translatedPayload.contains("\"errMsg\":\""+expectedErrMsg+"\""));
 	}
 }
diff --git a/src/test/resources/json/es-payload-translation.json b/src/test/resources/json/es-payload-translation.json
index e5290b0..58ed8f6 100644
--- a/src/test/resources/json/es-payload-translation.json
+++ b/src/test/resources/json/es-payload-translation.json
@@ -1,24 +1,16 @@
 {
-  "attr-translations": [
-    { 
-      "from": "\"data-type\":\"string\"",
-      "to": "\"data-type\":\"text\",\"fielddata\":true"
-    },    
+  "attr-translations": [       
     {
-      "from": "\"type\":\"string\",\"index\":\"analyzed\"",
-      "to": "\"type\":\"text\",\"index\":\"true\",\"fielddata\":true"
+      "query": "$..[?(@.type=='string' && @.index=='analyzed')]",
+      "update": {"type": "text", "index": true, "fielddata": true}
     },
     {
-      "from": "\"type\":\"string\",\"index\":\"not_analyzed\"",
-      "to": "\"type\":\"keyword\",\"index\":\"true\""
+      "query": "$..[?(@.type=='string' && @.index=='not_analyzed')]",
+      "update": {"type": "keyword", "index": true}
     },
     {
-      "from": "\"type\":\"string\"",
-      "to": "\"type\":\"text\",\"fielddata\":true"
-    },
-    {
-      "from": "searchable",
-      "to": "index"
+      "query": "$..[?(@.type=='string' && !@.index)]",
+      "update": {"type": "text", "fielddata": true}
     }
   ]
 }
\ No newline at end of file
diff --git a/src/test/resources/json/index-mapping.json b/src/test/resources/json/index-mapping.json
new file mode 100644
index 0000000..e447092
--- /dev/null
+++ b/src/test/resources/json/index-mapping.json
@@ -0,0 +1,28 @@
+{
+   "fields": [
+              {"name": "validationId", "type": "string", "searchable": false},
+              {"name": "validationTimestamp1", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"},
+              {"name": "entityId", "type": "nested"},
+              {"name": "entityType", "type": "string", "index": "analyzed"},
+              {"name": "entityLink", "type": "string"},
+              
+              {"name": "resourceVersion", "type": "string", "index": "not_analyzed"},
+              {"name": "violations", "type": "nested", "sub-fields": [
+                     {"name": "violationId", "type": "string"},
+                     {"name": "violationTimestamp", "type": "date", "format": "MMM d y HH:m:s||dd-MM-yyyy HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSSZZ||MM\/dd\/yyyy||yyyyMMdd'T'HHmmssZ"},            
+                     {"name": "modelName", "type": "string"},
+                     {"name": "category", "type": "string"},
+                     {"name": "severity", "type": "string"},
+                     {"name": "violationType", "type": "string"},
+                     {"name": "validationRule", "type": "string"},
+                     {"name": "violationDetails", "type": "nested", "sub-fields": [
+                           {"name": "entityId", "type": "nested"},
+                           {"name": "entityType", "type": "string"},
+                           {"name": "modelName", "type": "string"},
+                           {"name": "MISSING_REL", "type": "string"}
+                     ]},
+                     {"name": "errorMessage", "type": "string", "errMsg": "Sample error message for whitespace check"}
+              ]}
+       ]
+   
+}