Fix sync update flow

Issue-ID: AAI-1999
Change-Id: Id10b657236aff9ee1009ff0e1627a894e3881a87
Signed-off-by: fpaquett <francis.paquette@amdocs.com>

12 files changed, 57 insertions, 41 deletions

diff --git a/.gitignore b/.gitignore
index 5f4dc89..13c248c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@ target/
 logs/
 debug-logs/
 */appconfig-local/
+/.metadata/
diff --git a/sparkybe-onap-application/config/application-resources.properties b/sparkybe-onap-application/config/application-resources.properties
index e33cc4f..91906a3 100644
--- a/sparkybe-onap-application/config/application-resources.properties
+++ b/sparkybe-onap-application/config/application-resources.properties
@@ -8,4 +8,4 @@ resources.client-cert-password=1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10
 resources.trust-store=tomcat_keystore
 resources.maxSelfLinkTraversalDepth=2
 resources.domain=aai
-resources.delimitedShallowEntities=cloud-region,complex,vnf-image,image,flavor,availability-zone,tenant,network-profile,l-interface
\ No newline at end of file
+resources.delimitedShallowEntities=cloud-region,complex,vnf-image,image,flavor,availability-zone,tenant,network-profile,l-interface
diff --git a/sparkybe-onap-application/config/schemas/es_mappings.json b/sparkybe-onap-application/config/schemas/es_mappings.json
index 216e3d9..7eb25d0 100644
--- a/sparkybe-onap-application/config/schemas/es_mappings.json
+++ b/sparkybe-onap-application/config/schemas/es_mappings.json
@@ -16,7 +16,7 @@
             "type": "string",
             "analyzer": "ngram_analyzer"
         },
-        "crossEntityReferenceValues": {
+        "crossReferenceEntityValues": {
             "type": "string",
             "analyzer": "ngram_analyzer"
         },
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java
index dc6bdc0..70aaa93 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java
@@ -278,7 +278,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer
       try {
         versionNumber = NodeUtils.extractFieldValueFromObject(
             NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
-            "_version");
+            "etag");
       } catch (IOException exc) {
         String message =
             "Error extracting version number from response, aborting aggregation entity sync of "
@@ -303,7 +303,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer
           ArrayList<JsonNode> sourceObject = new ArrayList<JsonNode>();
           NodeUtils.extractObjectsByKey(
               NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
-              "_source", sourceObject);
+              "content", sourceObject);
 
           if (!sourceObject.isEmpty()) {
             String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false);
@@ -328,7 +328,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer
 
         	String requestPayload =
             		  searchServiceAdapter.buildBulkImportOperationRequest(schemaConfig.getIndexName(),
-                       ae.getId(),jsonPayload);
+                        versionNumber, ae.getId(),jsonPayload);
 
           NetworkTransaction transactionTracker = new NetworkTransaction();
           transactionTracker.setEntityType(esGetTxn.getEntityType());
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java
index 9481173..4b6b7a5 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java
@@ -684,7 +684,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer
       try {
         versionNumber = NodeUtils.extractFieldValueFromObject(
             NodeUtils.convertJsonStrToJsonNode(esGetResult.getOperationResult().getResult()),
-            "_version");
+            "etag");
       } catch (IOException exc) {
         LOG.error(AaiUiMsgs.ES_ABORT_CROSS_ENTITY_REF_SYNC, "version Number",
             icer.getEntityPrimaryKeyValue(), exc.getLocalizedMessage());
@@ -707,7 +707,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer
           ArrayList<JsonNode> sourceObject = new ArrayList<JsonNode>();
           NodeUtils.extractObjectsByKey(
               NodeUtils.convertJsonStrToJsonNode(esGetResult.getOperationResult().getResult()),
-              "_source", sourceObject);
+              "content", sourceObject);
 
           if (!sourceObject.isEmpty()) {
             String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false);
@@ -729,7 +729,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer
         if (versionNumber != null && jsonPayload != null) {
 
         	String requestPayload = searchServiceAdapter.buildBulkImportOperationRequest(getIndexName(),
-                    icer.getId(), jsonPayload);
+              icer.getId(), versionNumber, jsonPayload);
 
           NetworkTransaction transactionTracker = new NetworkTransaction();
           transactionTracker.setEntityType(esGetResult.getEntityType());
@@ -744,7 +744,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer
                     esWorkOnHand.decrementAndGet();
 
                     if (error != null) {
-                      LOG.error(AaiUiMsgs.ES_CROSS_ENTITY_REF_PUT, error.getLocalizedMessage());
+                       LOG.error(AaiUiMsgs.ES_CROSS_ENTITY_REF_PUT, error.getLocalizedMessage()); 
                     } else {
                       updateElasticSearchCounters(result);
                       processStoreDocumentResult(result, esGetResult, icer);
@@ -871,23 +871,6 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer
 
     IndexableCrossEntityReference icer = new IndexableCrossEntityReference();
 
-    icer.setEntityType(resultDescriptor.getEntityName());
-
-    List<String> primaryKeyValues = new ArrayList<String>();
-    String pkeyValue = null;
-
-    for (String keyName : resultDescriptor.getPrimaryKeyAttributeNames()) {
-      pkeyValue = NodeUtils.getNodeFieldAsText(entityNode, keyName);
-      if (pkeyValue != null) {
-        primaryKeyValues.add(pkeyValue);
-      } else {
-        LOG.warn(AaiUiMsgs.ES_PKEYVALUE_NULL, resultDescriptor.getEntityName());
-      }
-    }
-
-    final String primaryCompositeKeyValue = NodeUtils.concatArray(primaryKeyValues, "/");
-    icer.setEntityPrimaryKeyValue(primaryCompositeKeyValue);
-
     return icer;
 
   }
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java
index a00aa75..aab7e29 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java
@@ -45,8 +45,8 @@ public class SearchServiceAdapter {
   private static final String SUGGEST_QUERY = "suggest";
   private static final String BULK_API = "bulk";
   private static final String DOCUMENT_EDNPOINT = "documents";
-  private static final String SEARH_SERVICE_BULK_TEMPLATE =
-	      "{\"create\":{\"metaData\":{\"url\":\"%s\"},\"document\":\"%s\"}}\n";
+  private static final String SEARCH_SERVICE_BULK_TEMPLATE =
+	      "[{\"update\":{\"metaData\":{\"url\":\"%s\", \"etag\":\"%s\"},\"document\":%s}}]\n";
   
   private static final String SEARH_SERVICE_SINGLE_ENTITY_TEMPLATE =
 	      "{\"queries\":[{\"must\":{\"match\":{\"field\":\"_id\",\"value\":\"%s\"}}}]}\n";
@@ -145,12 +145,12 @@ public class SearchServiceAdapter {
     return headers;
   }
   
-public String buildBulkImportOperationRequest(String indexName, String id, String payload){
+public String buildBulkImportOperationRequest(String indexName, String id, String version, String payload){
 	  
 	  StringBuilder requestPayload = new StringBuilder(128);
-	  String SearchTarget = buildSearchServiceDocUrl(indexName,id);
+	  String SearchTarget = buildSearchServiceDocBulkPayloadUrl(indexName,id);
 	  
-	  requestPayload.append(String.format(SEARH_SERVICE_BULK_TEMPLATE,SearchTarget,payload));
+	  requestPayload.append(String.format(SEARCH_SERVICE_BULK_TEMPLATE,SearchTarget,version, payload));
 	  return requestPayload.toString();   
   }
 
@@ -177,12 +177,17 @@ public String buildBulkImportOperationRequest(String indexName, String id, Strin
   }
   
   public String buildSearchServiceDocUrl(String indexName,String api) {
-	  
+
 	  return String.format("https://%s:%s/services/search-data-service/%s/search/indexes/%s/%s/%s",
 		      endpointConfig.getEndpointIpAddress(), endpointConfig.getEndpointServerPort(),
 		        serviceApiVersion, indexName,DOCUMENT_EDNPOINT, api);
   }
   
+  public String buildSearchServiceDocBulkPayloadUrl(String indexName,String api) {
+
+	  return String.format("/services/search-data-service/%s/search/indexes/%s/%s/%s",
+		        serviceApiVersion, indexName,DOCUMENT_EDNPOINT, api);
+  }
   
   public String buildSearchServiceCreateDocApi(String indexName){
 	  
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java
index 0bad15a..c85980a 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java
@@ -21,17 +21,21 @@
 package org.onap.aai.sparky.sync.entity;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 
 import org.onap.aai.sparky.util.NodeUtils;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.core.JsonProcessingException;
-
+import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
+import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
 
 /**
  * The Class IndexableCrossEntityReference.
  */
-
+@JsonInclude(Include.NON_NULL)
 public class IndexableCrossEntityReference extends IndexableEntity implements IndexDocument {
 
   protected String crossReferenceEntityValues;
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java
index 1937891..77c1a69 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java
@@ -395,7 +395,7 @@ public class ViewInspectSearchProvider implements SearchProvider {
   
   private static final String VIUI_SEARCH_TEMPLATE =
       "{ " + "\"results-start\": 0," + "\"results-size\": %d," + "\"queries\": [{" + "\"must\": {"
-          + "\"match\": {" + "\"field\": \"entityType searchTags crossEntityReferenceValues\","
+          + "\"match\": {" + "\"field\": \"entityType searchTags crossReferenceEntityValues\","
           + "\"value\": \"%s\"," + "\"operator\": \"and\", "
           + "\"analyzer\": \"whitespace_analyzer\"" + "}" + "}" + "}]" + "}";
   
diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java
index 4877ad0..bc5e122 100644
--- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java
+++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java
@@ -393,7 +393,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer
       try {
         versionNumber = NodeUtils.extractFieldValueFromObject(
             NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
-            "_version");
+            "etag");
       } catch (IOException exc) {
         String message =
             "Error extracting version number from response, aborting searchable entity sync of "
@@ -418,7 +418,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer
           ArrayList<JsonNode> sourceObject = new ArrayList<JsonNode>();
           NodeUtils.extractObjectsByKey(
               NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
-              "_source", sourceObject);
+              "content", sourceObject);
 
           if (!sourceObject.isEmpty()) {
             String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false);
@@ -442,7 +442,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer
         if (versionNumber != null && jsonPayload != null) {
 
         	String requestPayload = searchServiceAdapter.buildBulkImportOperationRequest(getIndexName(),
-                    se.getId(), jsonPayload);
+               se.getId(), versionNumber, jsonPayload);
 
           NetworkTransaction transactionTracker = new NetworkTransaction();
           transactionTracker.setEntityType(esGetTxn.getEntityType());
diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java
index 2366439..0721fd2 100644
--- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java
+++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java
@@ -18,6 +18,8 @@ import org.mockito.Mockito;
 import org.onap.aai.restclient.client.OperationResult;
 import org.onap.aai.sparky.config.oxm.CrossEntityReferenceLookup;
 import org.onap.aai.sparky.config.oxm.OxmEntityDescriptor;
+import org.onap.aai.sparky.config.oxm.CrossEntityReferenceDescriptor;
+import org.onap.aai.sparky.config.oxm.CrossEntityReference;
 import org.onap.aai.sparky.config.oxm.OxmEntityLookup;
 import org.onap.aai.sparky.config.oxm.OxmModelProcessor;
 import org.onap.aai.sparky.config.oxm.SearchableEntityLookup;
@@ -144,11 +146,28 @@ public class CrossEntityReferenceSynchronizerTest {
 
     cerLookup = new CrossEntityReferenceLookup();
     processors.add(cerLookup);
-    
+
+    Map<String, CrossEntityReferenceDescriptor> crossReferenceEntityDescriptors =
+            new HashMap<String, CrossEntityReferenceDescriptor>();
+
+    CrossEntityReferenceDescriptor crossReferenceEntiDescriptor = new CrossEntityReferenceDescriptor();
+    CrossEntityReference crossEntityReference = new CrossEntityReference();
+    crossEntityReference.setTargetEntityType("service-instance");
+    List<String> refEntities = new ArrayList<String>();
+    refEntities.add("service-type");
+    crossEntityReference.setReferenceAttributes(refEntities);
+
+    List<String> crossEntityPkeyNames = new ArrayList<String>();
+    crossEntityPkeyNames.add("service-type");
+    crossReferenceEntiDescriptor.setPrimaryKeyAttributeNames(crossEntityPkeyNames);
+
+    crossReferenceEntiDescriptor.setCrossEntityReference(crossEntityReference);
+    crossReferenceEntiDescriptor.setEntityName("service-subscription");
+    crossReferenceEntityDescriptors.put("service-subscription", crossReferenceEntiDescriptor);
+    cerLookup.setCrossReferenceEntityDescriptors(crossReferenceEntityDescriptors);
+
     searchableEntityLookup = new SearchableEntityLookup();
     processors.add(searchableEntityLookup);
-    
-    
 
     aaiRestEndPointConfig = new RestEndpointConfig();
     aaiRestEndPointConfig.setNumRequestRetries(5);
diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java
index 59db5c2..c8bb33a 100644
--- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java
+++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java
@@ -44,6 +44,9 @@ public class SearchServiceAdapterTest {
         searchServiceAdapter.buildSearchServiceUrlForApi("searchentity-localhost", "2.0"));
     assertNotNull(searchServiceAdapter.buildSuggestServiceQueryUrl("searchentity-localhost"));
 
+    assertEquals(searchServiceAdapter.buildBulkImportOperationRequest("testIndex", "1", "4", "{\"payload\":\"value\"}"),
+       "[{\"update\":{\"metaData\":{\"url\":\"/services/search-data-service/2.0/search/indexes/testIndex/documents/1\", "
+       + "\"etag\":\"4\"},\"document\":{\"payload\":\"value\"}}}]\n");
 
   }
 
diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java
index a7c34d0..451bc98 100644
--- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java
+++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java
@@ -69,4 +69,5 @@ public class IndexableCrossEntityReferenceTest {
     System.out.println(icer.getAsJson());
 
   }
+
 }