From d89597a79b75aa6624fe4ff939e13a21d8b99a03 Mon Sep 17 00:00:00 2001 From: fpaquett Date: Wed, 12 Dec 2018 08:15:38 -0500 Subject: Fix sync update flow Issue-ID: AAI-1999 Change-Id: Id10b657236aff9ee1009ff0e1627a894e3881a87 Signed-off-by: fpaquett --- .gitignore | 1 + .../config/application-resources.properties | 2 +- .../config/schemas/es_mappings.json | 2 +- .../aggregation/sync/AggregationSynchronizer.java | 6 +++--- .../sync/CrossEntityReferenceSynchronizer.java | 25 ++++------------------ .../aai/sparky/search/SearchServiceAdapter.java | 17 +++++++++------ .../sync/entity/IndexableCrossEntityReference.java | 8 +++++-- .../search/ViewInspectSearchProvider.java | 2 +- .../sync/ViewInspectEntitySynchronizer.java | 6 +++--- .../sync/CrossEntityReferenceSynchronizerTest.java | 25 +++++++++++++++++++--- .../sparky/search/SearchServiceAdapterTest.java | 3 +++ .../IndexableCrossEntityReferenceTest.java | 1 + 12 files changed, 57 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 5f4dc89..13c248c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ target/ logs/ debug-logs/ */appconfig-local/ +/.metadata/ diff --git a/sparkybe-onap-application/config/application-resources.properties b/sparkybe-onap-application/config/application-resources.properties index e33cc4f..91906a3 100644 --- a/sparkybe-onap-application/config/application-resources.properties +++ b/sparkybe-onap-application/config/application-resources.properties @@ -8,4 +8,4 @@ resources.client-cert-password=1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 resources.trust-store=tomcat_keystore resources.maxSelfLinkTraversalDepth=2 resources.domain=aai -resources.delimitedShallowEntities=cloud-region,complex,vnf-image,image,flavor,availability-zone,tenant,network-profile,l-interface \ No newline at end of file +resources.delimitedShallowEntities=cloud-region,complex,vnf-image,image,flavor,availability-zone,tenant,network-profile,l-interface diff --git a/sparkybe-onap-application/config/schemas/es_mappings.json b/sparkybe-onap-application/config/schemas/es_mappings.json index 216e3d9..7eb25d0 100644 --- a/sparkybe-onap-application/config/schemas/es_mappings.json +++ b/sparkybe-onap-application/config/schemas/es_mappings.json @@ -16,7 +16,7 @@ "type": "string", "analyzer": "ngram_analyzer" }, - "crossEntityReferenceValues": { + "crossReferenceEntityValues": { "type": "string", "analyzer": "ngram_analyzer" }, diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java index dc6bdc0..70aaa93 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/aggregation/sync/AggregationSynchronizer.java @@ -278,7 +278,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer try { versionNumber = NodeUtils.extractFieldValueFromObject( NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()), - "_version"); + "etag"); } catch (IOException exc) { String message = "Error extracting version number from response, aborting aggregation entity sync of " @@ -303,7 +303,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer ArrayList sourceObject = new ArrayList(); NodeUtils.extractObjectsByKey( NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()), - "_source", sourceObject); + "content", sourceObject); if (!sourceObject.isEmpty()) { String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false); @@ -328,7 +328,7 @@ public class AggregationSynchronizer extends AbstractEntitySynchronizer String requestPayload = searchServiceAdapter.buildBulkImportOperationRequest(schemaConfig.getIndexName(), - ae.getId(),jsonPayload); + versionNumber, ae.getId(),jsonPayload); NetworkTransaction transactionTracker = new NetworkTransaction(); transactionTracker.setEntityType(esGetTxn.getEntityType()); diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java index 9481173..4b6b7a5 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizer.java @@ -684,7 +684,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer try { versionNumber = NodeUtils.extractFieldValueFromObject( NodeUtils.convertJsonStrToJsonNode(esGetResult.getOperationResult().getResult()), - "_version"); + "etag"); } catch (IOException exc) { LOG.error(AaiUiMsgs.ES_ABORT_CROSS_ENTITY_REF_SYNC, "version Number", icer.getEntityPrimaryKeyValue(), exc.getLocalizedMessage()); @@ -707,7 +707,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer ArrayList sourceObject = new ArrayList(); NodeUtils.extractObjectsByKey( NodeUtils.convertJsonStrToJsonNode(esGetResult.getOperationResult().getResult()), - "_source", sourceObject); + "content", sourceObject); if (!sourceObject.isEmpty()) { String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false); @@ -729,7 +729,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer if (versionNumber != null && jsonPayload != null) { String requestPayload = searchServiceAdapter.buildBulkImportOperationRequest(getIndexName(), - icer.getId(), jsonPayload); + icer.getId(), versionNumber, jsonPayload); NetworkTransaction transactionTracker = new NetworkTransaction(); transactionTracker.setEntityType(esGetResult.getEntityType()); @@ -744,7 +744,7 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer esWorkOnHand.decrementAndGet(); if (error != null) { - LOG.error(AaiUiMsgs.ES_CROSS_ENTITY_REF_PUT, error.getLocalizedMessage()); + LOG.error(AaiUiMsgs.ES_CROSS_ENTITY_REF_PUT, error.getLocalizedMessage()); } else { updateElasticSearchCounters(result); processStoreDocumentResult(result, esGetResult, icer); @@ -871,23 +871,6 @@ public class CrossEntityReferenceSynchronizer extends AbstractEntitySynchronizer IndexableCrossEntityReference icer = new IndexableCrossEntityReference(); - icer.setEntityType(resultDescriptor.getEntityName()); - - List primaryKeyValues = new ArrayList(); - String pkeyValue = null; - - for (String keyName : resultDescriptor.getPrimaryKeyAttributeNames()) { - pkeyValue = NodeUtils.getNodeFieldAsText(entityNode, keyName); - if (pkeyValue != null) { - primaryKeyValues.add(pkeyValue); - } else { - LOG.warn(AaiUiMsgs.ES_PKEYVALUE_NULL, resultDescriptor.getEntityName()); - } - } - - final String primaryCompositeKeyValue = NodeUtils.concatArray(primaryKeyValues, "/"); - icer.setEntityPrimaryKeyValue(primaryCompositeKeyValue); - return icer; } diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java index a00aa75..aab7e29 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/search/SearchServiceAdapter.java @@ -45,8 +45,8 @@ public class SearchServiceAdapter { private static final String SUGGEST_QUERY = "suggest"; private static final String BULK_API = "bulk"; private static final String DOCUMENT_EDNPOINT = "documents"; - private static final String SEARH_SERVICE_BULK_TEMPLATE = - "{\"create\":{\"metaData\":{\"url\":\"%s\"},\"document\":\"%s\"}}\n"; + private static final String SEARCH_SERVICE_BULK_TEMPLATE = + "[{\"update\":{\"metaData\":{\"url\":\"%s\", \"etag\":\"%s\"},\"document\":%s}}]\n"; private static final String SEARH_SERVICE_SINGLE_ENTITY_TEMPLATE = "{\"queries\":[{\"must\":{\"match\":{\"field\":\"_id\",\"value\":\"%s\"}}}]}\n"; @@ -145,12 +145,12 @@ public class SearchServiceAdapter { return headers; } -public String buildBulkImportOperationRequest(String indexName, String id, String payload){ +public String buildBulkImportOperationRequest(String indexName, String id, String version, String payload){ StringBuilder requestPayload = new StringBuilder(128); - String SearchTarget = buildSearchServiceDocUrl(indexName,id); + String SearchTarget = buildSearchServiceDocBulkPayloadUrl(indexName,id); - requestPayload.append(String.format(SEARH_SERVICE_BULK_TEMPLATE,SearchTarget,payload)); + requestPayload.append(String.format(SEARCH_SERVICE_BULK_TEMPLATE,SearchTarget,version, payload)); return requestPayload.toString(); } @@ -177,12 +177,17 @@ public String buildBulkImportOperationRequest(String indexName, String id, Strin } public String buildSearchServiceDocUrl(String indexName,String api) { - + return String.format("https://%s:%s/services/search-data-service/%s/search/indexes/%s/%s/%s", endpointConfig.getEndpointIpAddress(), endpointConfig.getEndpointServerPort(), serviceApiVersion, indexName,DOCUMENT_EDNPOINT, api); } + public String buildSearchServiceDocBulkPayloadUrl(String indexName,String api) { + + return String.format("/services/search-data-service/%s/search/indexes/%s/%s/%s", + serviceApiVersion, indexName,DOCUMENT_EDNPOINT, api); + } public String buildSearchServiceCreateDocApi(String indexName){ diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java index 0bad15a..c85980a 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/sync/entity/IndexableCrossEntityReference.java @@ -21,17 +21,21 @@ package org.onap.aai.sparky.sync.entity; import java.util.ArrayList; +import java.util.Arrays; import org.onap.aai.sparky.util.NodeUtils; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; - +import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter; +import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; /** * The Class IndexableCrossEntityReference. */ - +@JsonInclude(Include.NON_NULL) public class IndexableCrossEntityReference extends IndexableEntity implements IndexDocument { protected String crossReferenceEntityValues; diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java index 1937891..77c1a69 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewandinspect/search/ViewInspectSearchProvider.java @@ -395,7 +395,7 @@ public class ViewInspectSearchProvider implements SearchProvider { private static final String VIUI_SEARCH_TEMPLATE = "{ " + "\"results-start\": 0," + "\"results-size\": %d," + "\"queries\": [{" + "\"must\": {" - + "\"match\": {" + "\"field\": \"entityType searchTags crossEntityReferenceValues\"," + + "\"match\": {" + "\"field\": \"entityType searchTags crossReferenceEntityValues\"," + "\"value\": \"%s\"," + "\"operator\": \"and\", " + "\"analyzer\": \"whitespace_analyzer\"" + "}" + "}" + "}]" + "}"; diff --git a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java index 4877ad0..bc5e122 100644 --- a/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java +++ b/sparkybe-onap-service/src/main/java/org/onap/aai/sparky/viewinspect/sync/ViewInspectEntitySynchronizer.java @@ -393,7 +393,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer try { versionNumber = NodeUtils.extractFieldValueFromObject( NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()), - "_version"); + "etag"); } catch (IOException exc) { String message = "Error extracting version number from response, aborting searchable entity sync of " @@ -418,7 +418,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer ArrayList sourceObject = new ArrayList(); NodeUtils.extractObjectsByKey( NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()), - "_source", sourceObject); + "content", sourceObject); if (!sourceObject.isEmpty()) { String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false); @@ -442,7 +442,7 @@ public class ViewInspectEntitySynchronizer extends AbstractEntitySynchronizer if (versionNumber != null && jsonPayload != null) { String requestPayload = searchServiceAdapter.buildBulkImportOperationRequest(getIndexName(), - se.getId(), jsonPayload); + se.getId(), versionNumber, jsonPayload); NetworkTransaction transactionTracker = new NetworkTransaction(); transactionTracker.setEntityType(esGetTxn.getEntityType()); diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java index 2366439..0721fd2 100644 --- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java +++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/crossentityreference/sync/CrossEntityReferenceSynchronizerTest.java @@ -18,6 +18,8 @@ import org.mockito.Mockito; import org.onap.aai.restclient.client.OperationResult; import org.onap.aai.sparky.config.oxm.CrossEntityReferenceLookup; import org.onap.aai.sparky.config.oxm.OxmEntityDescriptor; +import org.onap.aai.sparky.config.oxm.CrossEntityReferenceDescriptor; +import org.onap.aai.sparky.config.oxm.CrossEntityReference; import org.onap.aai.sparky.config.oxm.OxmEntityLookup; import org.onap.aai.sparky.config.oxm.OxmModelProcessor; import org.onap.aai.sparky.config.oxm.SearchableEntityLookup; @@ -144,11 +146,28 @@ public class CrossEntityReferenceSynchronizerTest { cerLookup = new CrossEntityReferenceLookup(); processors.add(cerLookup); - + + Map crossReferenceEntityDescriptors = + new HashMap(); + + CrossEntityReferenceDescriptor crossReferenceEntiDescriptor = new CrossEntityReferenceDescriptor(); + CrossEntityReference crossEntityReference = new CrossEntityReference(); + crossEntityReference.setTargetEntityType("service-instance"); + List refEntities = new ArrayList(); + refEntities.add("service-type"); + crossEntityReference.setReferenceAttributes(refEntities); + + List crossEntityPkeyNames = new ArrayList(); + crossEntityPkeyNames.add("service-type"); + crossReferenceEntiDescriptor.setPrimaryKeyAttributeNames(crossEntityPkeyNames); + + crossReferenceEntiDescriptor.setCrossEntityReference(crossEntityReference); + crossReferenceEntiDescriptor.setEntityName("service-subscription"); + crossReferenceEntityDescriptors.put("service-subscription", crossReferenceEntiDescriptor); + cerLookup.setCrossReferenceEntityDescriptors(crossReferenceEntityDescriptors); + searchableEntityLookup = new SearchableEntityLookup(); processors.add(searchableEntityLookup); - - aaiRestEndPointConfig = new RestEndpointConfig(); aaiRestEndPointConfig.setNumRequestRetries(5); diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java index 59db5c2..c8bb33a 100644 --- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java +++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/search/SearchServiceAdapterTest.java @@ -44,6 +44,9 @@ public class SearchServiceAdapterTest { searchServiceAdapter.buildSearchServiceUrlForApi("searchentity-localhost", "2.0")); assertNotNull(searchServiceAdapter.buildSuggestServiceQueryUrl("searchentity-localhost")); + assertEquals(searchServiceAdapter.buildBulkImportOperationRequest("testIndex", "1", "4", "{\"payload\":\"value\"}"), + "[{\"update\":{\"metaData\":{\"url\":\"/services/search-data-service/2.0/search/indexes/testIndex/documents/1\", " + + "\"etag\":\"4\"},\"document\":{\"payload\":\"value\"}}}]\n"); } diff --git a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java index a7c34d0..451bc98 100644 --- a/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java +++ b/sparkybe-onap-service/src/test/java/org/onap/aai/sparky/synchronizer/IndexableCrossEntityReferenceTest.java @@ -69,4 +69,5 @@ public class IndexableCrossEntityReferenceTest { System.out.println(icer.getAsJson()); } + } -- cgit 1.2.3-korg