summaryrefslogtreecommitdiffstats
path: root/src/test/java/org/onap
diff options
context:
space:
mode:
authorda490c <dave.adams@amdocs.com>2018-03-08 12:54:22 -0500
committerda490c <dave.adams@amdocs.com>2018-03-08 12:59:30 -0500
commite5da3da7d7e0430a8d8d5c7c4a031c3ba557e247 (patch)
tree39a6272deea7c39d01820342e41f3923fe2f54fe /src/test/java/org/onap
parent629fdccb42a58348b6a691ab736ba1600ea813a3 (diff)
Integrate Sparky-BE with Gizmo
Issue-ID: AAI-847 Change-Id: I8c61e681c4f427f6e567dd8498b6f5f7baa19c85 Signed-off-by: da490c <dave.adams@amdocs.com>
Diffstat (limited to 'src/test/java/org/onap')
-rw-r--r--src/test/java/org/onap/aai/sparky/config/oxm/OxmEntityContainerLookup.java99
-rw-r--r--src/test/java/org/onap/aai/sparky/synchronizer/GizmoEntitySummarizer.java251
-rw-r--r--src/test/java/org/onap/aai/sparky/util/OxmModelAndProcessorHelper.java225
-rw-r--r--src/test/java/org/onap/aai/sparky/viewandinspect/BaseVisualizationServiceTest.java11
-rw-r--r--src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoEntitySynchronizer.java792
-rw-r--r--src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoSyncController.java106
6 files changed, 1376 insertions, 108 deletions
diff --git a/src/test/java/org/onap/aai/sparky/config/oxm/OxmEntityContainerLookup.java b/src/test/java/org/onap/aai/sparky/config/oxm/OxmEntityContainerLookup.java
new file mode 100644
index 0000000..7d55e4d
--- /dev/null
+++ b/src/test/java/org/onap/aai/sparky/config/oxm/OxmEntityContainerLookup.java
@@ -0,0 +1,99 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+package org.onap.aai.sparky.config.oxm;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.eclipse.persistence.dynamic.DynamicType;
+import org.eclipse.persistence.internal.oxm.mappings.Descriptor;
+import org.eclipse.persistence.jaxb.dynamic.DynamicJAXBContext;
+
+public class OxmEntityContainerLookup implements OxmModelProcessor {
+
+ private Collection<String> searchableEntityGroups;
+ private Collection<String> entityContainers;
+
+ public OxmEntityContainerLookup() {
+ searchableEntityGroups = new ArrayList<String>();
+ entityContainers = new ArrayList<String>();
+ }
+
+ @Override
+ public void processOxmModel(DynamicJAXBContext jaxbContext) {
+
+ @SuppressWarnings("rawtypes")
+ List<Descriptor> descriptorsList = jaxbContext.getXMLContext().getDescriptors();
+
+ for (@SuppressWarnings("rawtypes")
+ Descriptor desc : descriptorsList) {
+
+ DynamicType entity = jaxbContext.getDynamicType(desc.getAlias());
+
+ @SuppressWarnings("unchecked")
+ Map<String, String> properties = entity.getDescriptor().getProperties();
+
+ if (properties != null) {
+
+ String container = properties.get("container");
+
+ if (container != null && !entityContainers.contains(container)) {
+
+ entityContainers.add(container);
+
+ if (properties.containsKey("searchable")) {
+ if (!searchableEntityGroups.contains(container)) {
+ searchableEntityGroups.add(container);
+ }
+ }
+ }
+
+ }
+
+ }
+
+ }
+
+ public Collection<String> getSearchableEntityGroups() {
+ return searchableEntityGroups;
+ }
+
+ public void setSearchableEntityGroups(Collection<String> searchableEntityGroups) {
+ this.searchableEntityGroups = searchableEntityGroups;
+ }
+
+ public Collection<String> getEntityContainers() {
+ return entityContainers;
+ }
+
+ public void setEntityContainers(Collection<String> entityContainers) {
+ this.entityContainers = entityContainers;
+ }
+
+ public boolean isEntityContainer(String entityType) {
+ return entityContainers.contains(entityType);
+ }
+
+}
diff --git a/src/test/java/org/onap/aai/sparky/synchronizer/GizmoEntitySummarizer.java b/src/test/java/org/onap/aai/sparky/synchronizer/GizmoEntitySummarizer.java
new file mode 100644
index 0000000..5ea5280
--- /dev/null
+++ b/src/test/java/org/onap/aai/sparky/synchronizer/GizmoEntitySummarizer.java
@@ -0,0 +1,251 @@
+package org.onap.aai.sparky.synchronizer;
+
+import static java.util.concurrent.CompletableFuture.supplyAsync;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Supplier;
+
+import org.onap.aai.cl.api.Logger;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.aai.restclient.client.OperationResult;
+import org.onap.aai.restclient.enums.RestAuthenticationMode;
+import org.onap.aai.sparky.config.oxm.OxmModelLoader;
+import org.onap.aai.sparky.dal.GizmoAdapter;
+import org.onap.aai.sparky.dal.exception.ElasticSearchOperationException;
+import org.onap.aai.sparky.dal.rest.RestClientConstructionException;
+import org.onap.aai.sparky.dal.rest.config.RestEndpointConfig;
+import org.onap.aai.sparky.logging.AaiUiMsgs;
+import org.onap.aai.sparky.util.NodeUtils;
+import org.onap.aai.sparky.util.OxmModelAndProcessorHelper;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+
+public class GizmoEntitySummarizer {
+
+ protected ObjectMapper mapper;
+ protected OxmModelLoader oxmModelLoader;
+ private static final Logger logger = LoggerFactory.getInstance().getLogger(GizmoEntitySummarizer.class);
+ protected ExecutorService gizmoExecutor;
+ protected GizmoAdapter gizmoAdapter;
+ protected OxmModelAndProcessorHelper oxmHelper;
+
+ /*
+ * We need to add another concept to the OxmModelLoader which is to generate
+ * a list of entity containers from the OXM JaxbContext
+ */
+
+ public GizmoEntitySummarizer()
+ throws ElasticSearchOperationException, IOException, RestClientConstructionException {
+
+ OxmModelAndProcessorHelper.API_VERSION_OVERRIDE = 11;
+
+ this.gizmoExecutor = NodeUtils.createNamedExecutor("GIZMO-WORKER", 5, logger);
+
+ oxmHelper = OxmModelAndProcessorHelper.getInstance();
+ this.oxmModelLoader = oxmHelper.getModelLoader();
+
+ this.mapper = new ObjectMapper();
+
+ RestEndpointConfig gizmoConfig = new RestEndpointConfig();
+
+ gizmoConfig.setEndpointIpAddress("10.147.138.153");
+ gizmoConfig.setEndpointServerPort("9520");
+ gizmoConfig.setNumRequestRetries(5);
+ gizmoConfig.setRestAuthenticationMode(RestAuthenticationMode.SSL_CERT);
+ gizmoConfig.setConnectTimeoutInMs(60000);
+ gizmoConfig.setReadTimeoutInMs(30000);
+ gizmoConfig.setCertFileName("client-cert-onap.p12");
+ gizmoConfig.setCertPassword("OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10");
+ gizmoConfig.setTruststoreFileName("synchronizer.jks");
+ gizmoConfig.setValidateServerCertChain(false);
+ gizmoConfig.setValidateServerHostname(false);
+
+ gizmoAdapter = new GizmoAdapter(oxmModelLoader, gizmoConfig);
+
+ gizmoAdapter.setInventoryBasePath("/services/inventory/v12/");
+ gizmoAdapter.setRelationshipsBasePath("/services/inventory/relationships/v12/");
+
+ }
+
+ private Map<String, Integer> getNumEntitiesPerType() {
+
+ Collection<String> containerTypes = oxmHelper.getOxmEntityContainerLookup().getEntityContainers();
+ Collection<String> links = new ArrayList<String>();
+ Map<String, Integer> entityTypeCounts = new TreeMap<String, Integer>();
+
+ final CountDownLatch latch = new CountDownLatch(containerTypes.size());
+
+ for (String entityType : containerTypes) {
+
+ supplyAsync(new Supplier<Void>() {
+
+ @Override
+ public Void get() {
+
+ OperationResult typeLinksResult = null;
+ try {
+ typeLinksResult = gizmoAdapter.queryGizmoWithRetries(
+ gizmoAdapter.getFullInventoryUrl(entityType), "application/json", 1);
+
+ if (typeLinksResult != null) {
+
+ if (typeLinksResult.wasSuccessful() && typeLinksResult.getResult() != null) {
+
+ JsonNode rootNode = mapper.readValue(typeLinksResult.getResult(), JsonNode.class);
+
+ if (rootNode.isArray()) {
+ ArrayNode arrayNode = (ArrayNode) rootNode;
+ entityTypeCounts.put(entityType, new Integer(arrayNode.size()));
+ } else {
+ entityTypeCounts.put(entityType, new Integer(-1));
+ }
+
+ } else {
+ // -1
+ entityTypeCounts.put(entityType, new Integer(-1));
+ }
+
+ }
+
+ } catch (Exception exc) {
+ entityTypeCounts.put(entityType, new Integer(-1));
+ }
+
+ return null;
+ }
+
+ }, gizmoExecutor).whenComplete((result, error) -> {
+
+ latch.countDown();
+
+ if (error != null) {
+ logger.error(AaiUiMsgs.ERROR_GENERIC,
+ "An error occurred getting data from AAI. Error = " + error.getMessage());
+ }
+
+ });
+
+ }
+
+ // System.out.println("self links size = " + selflinks.size());
+
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+
+ }
+
+ return entityTypeCounts;
+ }
+
+ private Map<String, Integer> getNumRelationshipsPerType() {
+
+ Map<String, Integer> entityTypeCounts = new TreeMap<String, Integer>();
+
+ final CountDownLatch latch = new CountDownLatch(1);
+
+ supplyAsync(new Supplier<Void>() {
+
+ @Override
+ public Void get() {
+
+ OperationResult typeLinksResult = null;
+ try {
+ typeLinksResult = gizmoAdapter.queryGizmoWithRetries(gizmoAdapter.getFullRelationshipUrl("has"),
+ "application/json", 1);
+
+ if (typeLinksResult != null) {
+
+ if (typeLinksResult.wasSuccessful() && typeLinksResult.getResult() != null) {
+
+ JsonNode rootNode = mapper.readValue(typeLinksResult.getResult(), JsonNode.class);
+
+ if (rootNode.isArray()) {
+ ArrayNode arrayNode = (ArrayNode) rootNode;
+ entityTypeCounts.put("has", new Integer(arrayNode.size()));
+ } else {
+ entityTypeCounts.put("has", new Integer(-1));
+ }
+
+ } else {
+ // -1
+ entityTypeCounts.put("has", new Integer(-1));
+ }
+
+ } else {
+ entityTypeCounts.put("has", new Integer(-1));
+ }
+
+ } catch (Exception exc) {
+ entityTypeCounts.put("has", new Integer(-1));
+ }
+
+ return null;
+ }
+
+ }, gizmoExecutor).whenComplete((result, error) -> {
+
+ latch.countDown();
+
+ if (error != null) {
+ logger.error(AaiUiMsgs.ERROR_GENERIC,
+ "An error occurred getting data from AAI. Error = " + error.getMessage());
+ }
+
+ });
+
+ // System.out.println("self links size = " + selflinks.size());
+
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+
+ }
+
+ return entityTypeCounts;
+ }
+
+ public void shutdown() {
+ this.gizmoExecutor.shutdown();
+ }
+
+ public static void main(String[] args)
+ throws ElasticSearchOperationException, IOException, RestClientConstructionException {
+
+ System.setProperty("CONFIG_HOME", "X:\\2018_dev\\OSEAAI\\gizmo_integration\\onap_sparky-be\\appconfig-local\\");
+ GizmoEntitySummarizer gizmoSummarizer = new GizmoEntitySummarizer();
+
+ Map<String, Integer> entityCounts = gizmoSummarizer.getNumEntitiesPerType();
+ Map<String, Integer> relationshipCounts = gizmoSummarizer.getNumRelationshipsPerType();
+ gizmoSummarizer.shutdown();
+
+ System.out.println("Gizmo Entities:");
+
+ for (Entry<String, Integer> entry : entityCounts.entrySet()) {
+ String key = entry.getKey();
+ Integer value = entry.getValue();
+
+ System.out.printf("\t%s : %d\n", key, value);
+ }
+
+ System.out.println("\nGizmo Relationships:");
+
+ for (Entry<String, Integer> entry : relationshipCounts.entrySet()) {
+ String key = entry.getKey();
+ Integer value = entry.getValue();
+
+ System.out.printf("\t%s : %d\n", key, value);
+ }
+
+ }
+
+} \ No newline at end of file
diff --git a/src/test/java/org/onap/aai/sparky/util/OxmModelAndProcessorHelper.java b/src/test/java/org/onap/aai/sparky/util/OxmModelAndProcessorHelper.java
index bf5df76..48c8290 100644
--- a/src/test/java/org/onap/aai/sparky/util/OxmModelAndProcessorHelper.java
+++ b/src/test/java/org/onap/aai/sparky/util/OxmModelAndProcessorHelper.java
@@ -5,6 +5,7 @@ import java.util.Set;
import org.onap.aai.sparky.config.oxm.CrossEntityReferenceLookup;
import org.onap.aai.sparky.config.oxm.GeoEntityLookup;
+import org.onap.aai.sparky.config.oxm.OxmEntityContainerLookup;
import org.onap.aai.sparky.config.oxm.OxmEntityLookup;
import org.onap.aai.sparky.config.oxm.OxmModelLoader;
import org.onap.aai.sparky.config.oxm.OxmModelProcessor;
@@ -13,109 +14,123 @@ import org.onap.aai.sparky.config.oxm.SuggestionEntityLookup;
import org.onap.aai.sparky.search.filters.config.FiltersConfig;
public class OxmModelAndProcessorHelper {
-
- private static final int API_VERSION_OVERRIDE = -1;
-
- private OxmModelLoader modelLoader;
- private Set<OxmModelProcessor> processors;
-
- private CrossEntityReferenceLookup crossEntityReferenceLookup;
- private GeoEntityLookup geoEntityLookup;
- private OxmEntityLookup oxmEntityLookup;
- private SearchableEntityLookup searchableEntityLookup;
- private SuggestionEntityLookup suggestionEntityLookup;
- private FiltersConfig filtersConfig;
-
- private static OxmModelAndProcessorHelper instance = null;
- private OxmModelAndProcessorHelper() {
-
- this.filtersConfig = new FiltersConfig(SparkyTestConstants.FILTERS_JSON_FILE, SparkyTestConstants.VIEWS_JSON_FILE);
-
- this.crossEntityReferenceLookup = new CrossEntityReferenceLookup();
- this.geoEntityLookup = new GeoEntityLookup();
- this.oxmEntityLookup = new OxmEntityLookup();
- this.searchableEntityLookup = new SearchableEntityLookup();
- this.suggestionEntityLookup = new SuggestionEntityLookup(filtersConfig);
-
- this.processors = new HashSet<OxmModelProcessor>();
- processors.add(crossEntityReferenceLookup);
- processors.add(geoEntityLookup);
- processors.add(oxmEntityLookup);
- processors.add(searchableEntityLookup);
- processors.add(suggestionEntityLookup);
-
- this.modelLoader = new OxmModelLoader(API_VERSION_OVERRIDE, processors);
- modelLoader.loadLatestOxmModel();
- }
-
- public static OxmModelAndProcessorHelper getInstance() {
- if (instance == null) {
- instance = new OxmModelAndProcessorHelper();
- }
- return instance;
- }
-
- public OxmModelLoader getModelLoader() {
- return modelLoader;
- }
-
- public void setModelLoader(OxmModelLoader modelLoader) {
- this.modelLoader = modelLoader;
- }
-
- public Set<OxmModelProcessor> getProcessors() {
- return processors;
- }
-
- public void setProcessors(Set<OxmModelProcessor> processors) {
- this.processors = processors;
- }
-
- public CrossEntityReferenceLookup getCrossEntityReferenceLookup() {
- return crossEntityReferenceLookup;
- }
-
- public void setCrossEntityReferenceLookup(CrossEntityReferenceLookup crossEntityReferenceLookup) {
- this.crossEntityReferenceLookup = crossEntityReferenceLookup;
- }
-
- public GeoEntityLookup getGeoEntityLookup() {
- return geoEntityLookup;
- }
-
- public void setGeoEntityLookup(GeoEntityLookup geoEntityLookup) {
- this.geoEntityLookup = geoEntityLookup;
- }
-
- public OxmEntityLookup getOxmEntityLookup() {
- return oxmEntityLookup;
- }
-
- public void setOxmEntityLookup(OxmEntityLookup oxmEntityLookup) {
- this.oxmEntityLookup = oxmEntityLookup;
- }
-
- public SearchableEntityLookup getSearchableEntityLookup() {
- return searchableEntityLookup;
- }
-
- public void setSearchableEntityLookup(SearchableEntityLookup searchableEntityLookup) {
- this.searchableEntityLookup = searchableEntityLookup;
- }
-
- public SuggestionEntityLookup getSuggestionEntityLookup() {
- return suggestionEntityLookup;
- }
-
- public void setSuggestionEntityLookup(SuggestionEntityLookup suggestionEntityLookup) {
- this.suggestionEntityLookup = suggestionEntityLookup;
- }
-
- public FiltersConfig getFiltersConfig() {
- return filtersConfig;
- }
-
- public void setFiltersConfig(FiltersConfig filtersConfig) {
- this.filtersConfig = filtersConfig;
- }
+
+ public static int API_VERSION_OVERRIDE = -1;
+
+ private OxmModelLoader modelLoader;
+ private Set<OxmModelProcessor> processors;
+
+ private CrossEntityReferenceLookup crossEntityReferenceLookup;
+ private GeoEntityLookup geoEntityLookup;
+ private OxmEntityLookup oxmEntityLookup;
+ private SearchableEntityLookup searchableEntityLookup;
+ private SuggestionEntityLookup suggestionEntityLookup;
+ private OxmEntityContainerLookup oxmEntityContainerLookup;
+ private FiltersConfig filtersConfig;
+
+ private static OxmModelAndProcessorHelper instance = null;
+
+ private OxmModelAndProcessorHelper() {
+
+ this.filtersConfig = new FiltersConfig(SparkyTestConstants.FILTERS_JSON_FILE,
+ SparkyTestConstants.VIEWS_JSON_FILE);
+
+ this.crossEntityReferenceLookup = new CrossEntityReferenceLookup();
+ this.geoEntityLookup = new GeoEntityLookup();
+ this.oxmEntityLookup = new OxmEntityLookup();
+ this.searchableEntityLookup = new SearchableEntityLookup();
+ this.suggestionEntityLookup = new SuggestionEntityLookup(filtersConfig);
+ this.oxmEntityContainerLookup = new OxmEntityContainerLookup();
+
+ this.processors = new HashSet<OxmModelProcessor>();
+ processors.add(crossEntityReferenceLookup);
+ processors.add(geoEntityLookup);
+ processors.add(oxmEntityLookup);
+ processors.add(searchableEntityLookup);
+ processors.add(suggestionEntityLookup);
+ processors.add(oxmEntityContainerLookup);
+
+ this.modelLoader = new OxmModelLoader(API_VERSION_OVERRIDE, processors);
+ modelLoader.loadLatestOxmModel();
+ }
+
+ public static OxmModelAndProcessorHelper getInstance() {
+ if (instance == null) {
+ instance = new OxmModelAndProcessorHelper();
+ }
+ return instance;
+ }
+
+ public OxmModelLoader getModelLoader() {
+ return modelLoader;
+ }
+
+ public void setModelLoader(OxmModelLoader modelLoader) {
+ this.modelLoader = modelLoader;
+ }
+
+ public Set<OxmModelProcessor> getProcessors() {
+ return processors;
+ }
+
+ public void setProcessors(Set<OxmModelProcessor> processors) {
+ this.processors = processors;
+ }
+
+ public CrossEntityReferenceLookup getCrossEntityReferenceLookup() {
+ return crossEntityReferenceLookup;
+ }
+
+ public void setCrossEntityReferenceLookup(CrossEntityReferenceLookup crossEntityReferenceLookup) {
+ this.crossEntityReferenceLookup = crossEntityReferenceLookup;
+ }
+
+ public GeoEntityLookup getGeoEntityLookup() {
+ return geoEntityLookup;
+ }
+
+ public void setGeoEntityLookup(GeoEntityLookup geoEntityLookup) {
+ this.geoEntityLookup = geoEntityLookup;
+ }
+
+ public OxmEntityLookup getOxmEntityLookup() {
+ return oxmEntityLookup;
+ }
+
+ public void setOxmEntityLookup(OxmEntityLookup oxmEntityLookup) {
+ this.oxmEntityLookup = oxmEntityLookup;
+ }
+
+ public SearchableEntityLookup getSearchableEntityLookup() {
+ return searchableEntityLookup;
+ }
+
+ public void setSearchableEntityLookup(SearchableEntityLookup searchableEntityLookup) {
+ this.searchableEntityLookup = searchableEntityLookup;
+ }
+
+ public SuggestionEntityLookup getSuggestionEntityLookup() {
+ return suggestionEntityLookup;
+ }
+
+ public void setSuggestionEntityLookup(SuggestionEntityLookup suggestionEntityLookup) {
+ this.suggestionEntityLookup = suggestionEntityLookup;
+ }
+
+ public FiltersConfig getFiltersConfig() {
+ return filtersConfig;
+ }
+
+ public void setFiltersConfig(FiltersConfig filtersConfig) {
+ this.filtersConfig = filtersConfig;
+ }
+
+ public OxmEntityContainerLookup getOxmEntityContainerLookup() {
+ return oxmEntityContainerLookup;
+ }
+
+ public void setOxmEntityContainerLookup(OxmEntityContainerLookup oxmEntityContainerLookup) {
+ this.oxmEntityContainerLookup = oxmEntityContainerLookup;
+ }
+
}
diff --git a/src/test/java/org/onap/aai/sparky/viewandinspect/BaseVisualizationServiceTest.java b/src/test/java/org/onap/aai/sparky/viewandinspect/BaseVisualizationServiceTest.java
index bc1a80d..e51c629 100644
--- a/src/test/java/org/onap/aai/sparky/viewandinspect/BaseVisualizationServiceTest.java
+++ b/src/test/java/org/onap/aai/sparky/viewandinspect/BaseVisualizationServiceTest.java
@@ -1,6 +1,6 @@
package org.onap.aai.sparky.viewandinspect;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Test;
@@ -9,6 +9,7 @@ import org.onap.aai.sparky.config.oxm.OxmEntityLookup;
import org.onap.aai.sparky.config.oxm.OxmModelLoader;
import org.onap.aai.sparky.dal.ActiveInventoryAdapter;
import org.onap.aai.sparky.dal.ElasticSearchAdapter;
+import org.onap.aai.sparky.dal.GizmoAdapter;
import org.onap.aai.sparky.subscription.config.SubscriptionConfig;
import org.onap.aai.sparky.sync.config.ElasticSearchEndpointConfig;
import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig;
@@ -26,13 +27,16 @@ public class BaseVisualizationServiceTest {
private ElasticSearchEndpointConfig endpointEConfig;
private ElasticSearchSchemaConfig schemaEConfig;
private OxmEntityLookup oxmEntityLookup;
+ private GizmoAdapter mockGizmoAdapter;
private BaseVisualizationService baseVisService;
@Before
public void init() throws Exception {
this.mockAaiAdapter = Mockito.mock(ActiveInventoryAdapter.class);
+ this.mockAaiAdapter = Mockito.mock(ActiveInventoryAdapter.class);
this.mockEsAdapter = Mockito.mock(ElasticSearchAdapter.class);
+ this.mockGizmoAdapter = Mockito.mock(GizmoAdapter.class);
this.visualizationConfigs = new VisualizationConfigs();
this.subConfig = new SubscriptionConfig();
this.endpointEConfig = new ElasticSearchEndpointConfig();
@@ -41,8 +45,9 @@ public class BaseVisualizationServiceTest {
OxmModelLoader modelLoader = OxmModelAndProcessorHelper.getInstance().getModelLoader();
- this.baseVisService = new BaseVisualizationService(modelLoader, visualizationConfigs, mockAaiAdapter,
- mockEsAdapter, endpointEConfig, schemaEConfig, 1, oxmEntityLookup, subConfig);
+ this.baseVisService = new BaseVisualizationService(modelLoader, visualizationConfigs,
+ mockAaiAdapter, mockGizmoAdapter, mockEsAdapter, endpointEConfig, schemaEConfig, 1,
+ oxmEntityLookup, subConfig);
}
@Test
diff --git a/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoEntitySynchronizer.java b/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoEntitySynchronizer.java
new file mode 100644
index 0000000..6d63a8a
--- /dev/null
+++ b/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoEntitySynchronizer.java
@@ -0,0 +1,792 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+package org.onap.aai.sparky.viewandinspect.sync;
+
+import static java.util.concurrent.CompletableFuture.supplyAsync;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Supplier;
+
+import org.onap.aai.cl.api.Logger;
+import org.onap.aai.cl.eelf.LoggerFactory;
+import org.onap.aai.cl.mdc.MdcContext;
+import org.onap.aai.restclient.client.OperationResult;
+import org.onap.aai.sparky.config.oxm.OxmEntityContainerLookup;
+import org.onap.aai.sparky.config.oxm.OxmEntityDescriptor;
+import org.onap.aai.sparky.config.oxm.OxmEntityLookup;
+import org.onap.aai.sparky.config.oxm.SearchableEntityLookup;
+import org.onap.aai.sparky.config.oxm.SearchableOxmEntityDescriptor;
+import org.onap.aai.sparky.dal.GizmoAdapter;
+import org.onap.aai.sparky.dal.NetworkTransaction;
+import org.onap.aai.sparky.dal.rest.HttpMethod;
+import org.onap.aai.sparky.logging.AaiUiMsgs;
+import org.onap.aai.sparky.sync.AbstractEntitySynchronizer;
+import org.onap.aai.sparky.sync.IndexSynchronizer;
+import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig;
+import org.onap.aai.sparky.sync.config.NetworkStatisticsConfig;
+import org.onap.aai.sparky.sync.entity.MergableEntity;
+import org.onap.aai.sparky.sync.entity.SearchableEntity;
+import org.onap.aai.sparky.sync.entity.SelfLinkDescriptor;
+import org.onap.aai.sparky.sync.enumeration.OperationState;
+import org.onap.aai.sparky.sync.enumeration.SynchronizerState;
+import org.onap.aai.sparky.sync.task.PerformElasticSearchPut;
+import org.onap.aai.sparky.sync.task.PerformElasticSearchRetrieval;
+import org.onap.aai.sparky.sync.task.PerformElasticSearchUpdate;
+import org.onap.aai.sparky.sync.task.PerformGizmoRetrieval;
+import org.onap.aai.sparky.util.NodeUtils;
+import org.slf4j.MDC;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+
+/**
+ * The Class SearchableEntitySynchronizer.
+ */
+public class ViewInspectGizmoEntitySynchronizer extends AbstractEntitySynchronizer
+ implements IndexSynchronizer {
+
+ /**
+ * The Class RetrySearchableEntitySyncContainer.
+ */
+ private class RetrySearchableEntitySyncContainer {
+ NetworkTransaction txn;
+ SearchableEntity se;
+
+ /**
+ * Instantiates a new retry searchable entity sync container.
+ *
+ * @param txn the txn
+ * @param se the se
+ */
+ public RetrySearchableEntitySyncContainer(NetworkTransaction txn, SearchableEntity se) {
+ this.txn = txn;
+ this.se = se;
+ }
+
+ public NetworkTransaction getNetworkTransaction() {
+ return txn;
+ }
+
+ public SearchableEntity getSearchableEntity() {
+ return se;
+ }
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getInstance().getLogger(ViewInspectGizmoEntitySynchronizer.class);
+
+ private boolean allWorkEnumerated;
+ private Deque<SelfLinkDescriptor> selflinks;
+ private Deque<RetrySearchableEntitySyncContainer> retryQueue;
+ private Map<String, Integer> retryLimitTracker;
+ protected ExecutorService esPutExecutor;
+ private OxmEntityLookup oxmEntityLookup;
+ private SearchableEntityLookup searchableEntityLookup;
+ private GizmoAdapter gizmoAdapter;
+ private OxmEntityContainerLookup entityContainerLookup;
+
+ /**
+ * Instantiates a new searchable entity synchronizer.
+ *
+ * @param indexName the index name
+ * @throws Exception the exception
+ */
+ public ViewInspectGizmoEntitySynchronizer(ElasticSearchSchemaConfig schemaConfig,
+ int internalSyncWorkers, int gizmoWorkers, int esWorkers, NetworkStatisticsConfig aaiStatConfig,
+ NetworkStatisticsConfig esStatConfig, OxmEntityLookup oxmEntityLookup,
+ SearchableEntityLookup searchableEntityLookup, OxmEntityContainerLookup entityContainerLookup) throws Exception {
+ super(LOG, "SES", internalSyncWorkers, gizmoWorkers, esWorkers, schemaConfig.getIndexName(),
+ aaiStatConfig, esStatConfig);
+
+ this.oxmEntityLookup = oxmEntityLookup;
+ this.searchableEntityLookup = searchableEntityLookup;
+ this.entityContainerLookup = entityContainerLookup;
+ this.allWorkEnumerated = false;
+ this.selflinks = new ConcurrentLinkedDeque<SelfLinkDescriptor>();
+ this.retryQueue = new ConcurrentLinkedDeque<RetrySearchableEntitySyncContainer>();
+ this.retryLimitTracker = new ConcurrentHashMap<String, Integer>();
+ this.synchronizerName = "Searchable Entity Synchronizer";
+ this.esPutExecutor = NodeUtils.createNamedExecutor("SES-ES-PUT", 5, LOG);
+ this.aaiEntityStats.intializeEntityCounters(
+ searchableEntityLookup.getSearchableEntityDescriptors().keySet());
+ this.esEntityStats.intializeEntityCounters(
+ searchableEntityLookup.getSearchableEntityDescriptors().keySet());
+ this.syncDurationInMs = -1;
+ }
+
+
+
+ public GizmoAdapter getGizmoAdapter() {
+ return gizmoAdapter;
+}
+
+
+
+public void setGizmoAdapter(GizmoAdapter gizmoAdapter) {
+ this.gizmoAdapter = gizmoAdapter;
+}
+
+
+
+/**
+ * Collect all the work.
+ *
+ * @return the operation state
+ */
+ private OperationState collectAllTheWork() {
+ final Map<String, String> contextMap = MDC.getCopyOfContextMap();
+
+ Collection<String> searchableEntityGroups = entityContainerLookup.getSearchableEntityGroups();
+
+ if (searchableEntityGroups.isEmpty()) {
+ LOG.error(AaiUiMsgs.ERROR_LOADING_OXM_SEARCHABLE_ENTITIES);
+ return OperationState.ERROR;
+ }
+
+
+ try {
+
+ /*
+ * launch a parallel async thread to process the documents for each entity-type (to max the
+ * of the configured executor anyway)
+ */
+
+ /*searchableEntityGroups = new ArrayList<String>();
+ searchableEntityGroups.add("pservers");*/
+
+ aaiWorkOnHand.set(searchableEntityGroups.size());
+
+ for (String searchableEntityGroup : searchableEntityGroups) {
+
+ supplyAsync(new Supplier<Void>() {
+
+ @Override
+ public Void get() {
+ MDC.setContextMap(contextMap);
+ OperationResult typeLinksResult = null;
+ try {
+ typeLinksResult = gizmoAdapter.getSelfLinksByEntityType(searchableEntityGroup);
+ aaiWorkOnHand.decrementAndGet();
+ processEntityTypeSelfLinks(typeLinksResult);
+ } catch (Exception exc) {
+
+ exc.printStackTrace();
+ }
+
+ return null;
+ }
+
+ }, aaiExecutor).whenComplete((result, error) -> {
+
+ if (error != null) {
+ LOG.error(AaiUiMsgs.ERROR_GENERIC,
+ "An error occurred getting data from AAI. Error = " + error.getMessage());
+ }
+ });
+
+ }
+
+ while (aaiWorkOnHand.get() != 0) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(AaiUiMsgs.WAIT_FOR_ALL_SELFLINKS_TO_BE_COLLECTED);
+ }
+
+ Thread.sleep(1000);
+ }
+
+ aaiWorkOnHand.set(selflinks.size());
+ allWorkEnumerated = true;
+ syncEntityTypes();
+
+ while (!isSyncDone()) {
+ performRetrySync();
+ Thread.sleep(1000);
+ }
+
+ /*
+ * Make sure we don't hang on to retries that failed which could cause issues during future
+ * syncs
+ */
+ retryLimitTracker.clear();
+
+ } catch (Exception exc) {
+ // TODO -> LOG, waht should be logged here?
+ }
+
+ return OperationState.OK;
+ }
+
+ /* (non-Javadoc)
+ * @see org.openecomp.sparky.synchronizer.IndexSynchronizer#doSync()
+ */
+ @Override
+ public OperationState doSync() {
+ this.syncDurationInMs = -1;
+ String txnID = NodeUtils.getRandomTxnId();
+ MdcContext.initialize(txnID, "SearchableEntitySynchronizer", "", "Sync", "");
+
+ resetCounters();
+ this.allWorkEnumerated = false;
+ syncStartedTimeStampInMs = System.currentTimeMillis();
+ collectAllTheWork();
+
+ return OperationState.OK;
+ }
+
+ /**
+ * Process entity type self links.
+ *
+ * @param operationResult the operation result
+ */
+ private void processEntityTypeSelfLinks(OperationResult operationResult) {
+
+ JsonNode rootNode = null;
+
+ final String jsonResult = operationResult.getResult();
+
+ if (jsonResult != null && jsonResult.length() > 0 && operationResult.wasSuccessful()) {
+
+ try {
+ rootNode = mapper.readTree(jsonResult);
+ } catch (IOException exc) {
+ String message = "Could not deserialize JSON (representing operation result) as node tree. "
+ + "Operation result = " + jsonResult + ". " + exc.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.JSON_PROCESSING_ERROR, message);
+ }
+
+ ArrayNode resultDataArrayNode = null;
+
+ if (rootNode.isArray()) {
+ resultDataArrayNode = (ArrayNode) rootNode;
+
+ Iterator<JsonNode> elementIterator = resultDataArrayNode.elements();
+ JsonNode element = null;
+
+ while (elementIterator.hasNext()) {
+ element = elementIterator.next();
+
+ final String id = NodeUtils.getNodeFieldAsText(element, "id");
+ final String type = NodeUtils.getNodeFieldAsText(element, "type");
+ final String url = NodeUtils.getNodeFieldAsText(element, "url");
+
+ String resourceLink;
+ try {
+ resourceLink = gizmoAdapter.getFullInventoryUrl(type + "/" + id);
+ selflinks.add(new SelfLinkDescriptor(NodeUtils.extractRawGizmoPathWithoutVersion(resourceLink), null, type));
+ } catch (Exception e) {
+ LOG.error(AaiUiMsgs.ERROR_GENERIC, "ERROR: Failed to determine resource link caused by " + e.getMessage());
+ }
+
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Sync entity types.
+ */
+ private void syncEntityTypes() {
+
+ while (selflinks.peek() != null) {
+
+ SelfLinkDescriptor linkDescriptor = selflinks.poll();
+ aaiWorkOnHand.decrementAndGet();
+
+ OxmEntityDescriptor descriptor = null;
+
+ if (linkDescriptor.getSelfLink() != null && linkDescriptor.getEntityType() != null) {
+
+ descriptor = oxmEntityLookup.getEntityDescriptors().get(linkDescriptor.getEntityType());
+
+ if (descriptor == null) {
+ LOG.error(AaiUiMsgs.MISSING_ENTITY_DESCRIPTOR, linkDescriptor.getEntityType());
+ continue;
+ }
+
+ NetworkTransaction txn = new NetworkTransaction();
+ txn.setDescriptor(descriptor);
+ txn.setLink(linkDescriptor.getSelfLink());
+ txn.setOperationType(HttpMethod.GET);
+ txn.setEntityType(linkDescriptor.getEntityType());
+
+ aaiWorkOnHand.incrementAndGet();
+
+ supplyAsync(new PerformGizmoRetrieval(txn, gizmoAdapter), aaiExecutor)
+ .whenComplete((result, error) -> {
+
+ aaiWorkOnHand.decrementAndGet();
+
+ if (error != null) {
+ LOG.error(AaiUiMsgs.AAI_RETRIEVAL_FAILED_GENERIC, error.getLocalizedMessage());
+ } else {
+ if (result == null) {
+ LOG.error(AaiUiMsgs.AAI_RETRIEVAL_FAILED_FOR_SELF_LINK,
+ linkDescriptor.getSelfLink());
+ } else {
+ updateActiveInventoryCounters(result);
+ fetchDocumentForUpsert(result);
+ }
+ }
+ });
+ }
+
+ }
+
+ }
+
+ /**
+ * Perform document upsert.
+ *
+ * @param esGetTxn the es get txn
+ * @param se the se
+ */
+ protected void performDocumentUpsert(NetworkTransaction esGetTxn, SearchableEntity se) {
+ /**
+ * <p>
+ * <ul>
+ * As part of the response processing we need to do the following:
+ * <li>1. Extract the version (if present), it will be the ETAG when we use the
+ * Search-Abstraction-Service
+ * <li>2. Spawn next task which is to do the PUT operation into elastic with or with the version
+ * tag
+ * <li>a) if version is null or RC=404, then standard put, no _update with version tag
+ * <li>b) if version != null, do PUT with _update?version= versionNumber in the URI to elastic
+ * </ul>
+ * </p>
+ */
+ String link = null;
+ try {
+ link = elasticSearchAdapter.buildElasticSearchGetDocUrl(getIndexName(), se.getId());
+ } catch (Exception exc) {
+ LOG.error(AaiUiMsgs.ES_LINK_UPSERT, exc.getLocalizedMessage());
+ return;
+ }
+
+ String versionNumber = null;
+ boolean wasEntryDiscovered = false;
+ if (esGetTxn.getOperationResult().getResultCode() == 404) {
+ LOG.info(AaiUiMsgs.ES_SIMPLE_PUT, se.getEntityPrimaryKeyValue());
+ } else if (esGetTxn.getOperationResult().getResultCode() == 200) {
+ wasEntryDiscovered = true;
+ try {
+ versionNumber = NodeUtils.extractFieldValueFromObject(
+ NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
+ "_version");
+ } catch (IOException exc) {
+ String message =
+ "Error extracting version number from response, aborting searchable entity sync of "
+ + se.getEntityPrimaryKeyValue() + ". Error - " + exc.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.ERROR_EXTRACTING_FROM_RESPONSE, message);
+ return;
+ }
+ } else {
+ /*
+ * Not being a 200 does not mean a failure. eg 201 is returned for created. TODO -> Should we
+ * return.
+ */
+ LOG.error(AaiUiMsgs.ES_OPERATION_RETURN_CODE,
+ String.valueOf(esGetTxn.getOperationResult().getResultCode()));
+ return;
+ }
+
+ try {
+ String jsonPayload = null;
+ if (wasEntryDiscovered) {
+ try {
+ ArrayList<JsonNode> sourceObject = new ArrayList<JsonNode>();
+ NodeUtils.extractObjectsByKey(
+ NodeUtils.convertJsonStrToJsonNode(esGetTxn.getOperationResult().getResult()),
+ "_source", sourceObject);
+
+ if (!sourceObject.isEmpty()) {
+ String responseSource = NodeUtils.convertObjectToJson(sourceObject.get(0), false);
+ MergableEntity me = mapper.readValue(responseSource, MergableEntity.class);
+ ObjectReader updater = mapper.readerForUpdating(me);
+ MergableEntity merged = updater.readValue(NodeUtils.convertObjectToJson(se,false));
+ jsonPayload = mapper.writeValueAsString(merged);
+ }
+ } catch (IOException exc) {
+ String message =
+ "Error extracting source value from response, aborting searchable entity sync of "
+ + se.getEntityPrimaryKeyValue() + ". Error - " + exc.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.ERROR_EXTRACTING_FROM_RESPONSE, message);
+ return;
+ }
+ } else {
+ jsonPayload = se.getAsJson();
+ }
+
+ if (wasEntryDiscovered) {
+ if (versionNumber != null && jsonPayload != null) {
+
+ String requestPayload = elasticSearchAdapter.buildBulkImportOperationRequest(getIndexName(),
+ "default", se.getId(), versionNumber, jsonPayload);
+
+ NetworkTransaction transactionTracker = new NetworkTransaction();
+ transactionTracker.setEntityType(esGetTxn.getEntityType());
+ transactionTracker.setDescriptor(esGetTxn.getDescriptor());
+ transactionTracker.setOperationType(HttpMethod.PUT);
+
+ esWorkOnHand.incrementAndGet();
+ supplyAsync(new PerformElasticSearchUpdate(elasticSearchAdapter.getBulkUrl(),
+ requestPayload, elasticSearchAdapter, transactionTracker), esPutExecutor)
+ .whenComplete((result, error) -> {
+
+ esWorkOnHand.decrementAndGet();
+
+ if (error != null) {
+ String message = "Searchable entity sync UPDATE PUT error - "
+ + error.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ } else {
+ updateElasticSearchCounters(result);
+ processStoreDocumentResult(result, esGetTxn, se);
+ }
+ });
+ }
+
+ } else {
+
+ if (link != null && jsonPayload != null) {
+
+ NetworkTransaction updateElasticTxn = new NetworkTransaction();
+ updateElasticTxn.setLink(link);
+ updateElasticTxn.setEntityType(esGetTxn.getEntityType());
+ updateElasticTxn.setDescriptor(esGetTxn.getDescriptor());
+ updateElasticTxn.setOperationType(HttpMethod.PUT);
+
+ esWorkOnHand.incrementAndGet();
+ supplyAsync(new PerformElasticSearchPut(jsonPayload, updateElasticTxn, elasticSearchAdapter),
+ esPutExecutor).whenComplete((result, error) -> {
+
+ esWorkOnHand.decrementAndGet();
+
+ if (error != null) {
+ String message =
+ "Searchable entity sync UPDATE PUT error - " + error.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ } else {
+ updateElasticSearchCounters(result);
+ processStoreDocumentResult(result, esGetTxn, se);
+ }
+ });
+ }
+ }
+ } catch (Exception exc) {
+ String message = "Exception caught during searchable entity sync PUT operation. Message - "
+ + exc.getLocalizedMessage();
+ LOG.error(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ }
+ }
+
+ /**
+ * Populate searchable entity document.
+ *
+ * @param doc the doc
+ * @param result the result
+ * @param resultDescriptor the result descriptor
+ * @throws JsonProcessingException the json processing exception
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ protected void populateSearchableEntityDocument(SearchableEntity doc, String result,
+ OxmEntityDescriptor resultDescriptor) throws JsonProcessingException, IOException {
+
+ doc.setEntityType(resultDescriptor.getEntityName());
+
+ JsonNode entityNode = mapper.readTree(result);
+
+ String id = NodeUtils.getNodeFieldAsText(entityNode, "id");
+ String type = NodeUtils.getNodeFieldAsText(entityNode, "type");
+ String url = NodeUtils.getNodeFieldAsText(entityNode, "url");
+
+ JsonNode properties = entityNode.get("properties");
+
+ Iterator<String> fieldNames = properties.fieldNames();
+
+
+
+ List<String> primaryKeyValues = new ArrayList<String>();
+ String pkeyValue = null;
+
+ SearchableOxmEntityDescriptor searchableDescriptor = searchableEntityLookup.getSearchableEntityDescriptors().get(resultDescriptor.getEntityName());
+
+ for (String keyName : searchableDescriptor.getPrimaryKeyAttributeNames()) {
+ pkeyValue = NodeUtils.getNodeFieldAsText(properties, keyName);
+ if (pkeyValue != null) {
+ primaryKeyValues.add(pkeyValue);
+ } else {
+ String message = "populateSearchableEntityDocument(), pKeyValue is null for entityType = "
+ + resultDescriptor.getEntityName();
+ LOG.warn(AaiUiMsgs.WARN_GENERIC, message);
+ }
+ }
+
+ final String primaryCompositeKeyValue = NodeUtils.concatArray(primaryKeyValues, "/");
+ doc.setEntityPrimaryKeyValue(primaryCompositeKeyValue);
+
+ final List<String> searchTagFields = searchableDescriptor.getSearchableAttributes();
+
+ /*
+ * Based on configuration, use the configured field names for this entity-Type to build a
+ * multi-value collection of search tags for elastic search entity search criteria.
+ */
+ for (String searchTagField : searchTagFields) {
+ String searchTagValue = NodeUtils.getNodeFieldAsText(properties, searchTagField);
+ if (searchTagValue != null && !searchTagValue.isEmpty()) {
+ doc.addSearchTagWithKey(searchTagValue, searchTagField);
+ }
+ }
+ }
+
+ /**
+ * Fetch document for upsert.
+ *
+ * @param txn the txn
+ */
+ private void fetchDocumentForUpsert(NetworkTransaction txn) {
+ if (!txn.getOperationResult().wasSuccessful()) {
+ String message = "Self link failure. Result - " + txn.getOperationResult().getResult();
+ LOG.error(AaiUiMsgs.ERROR_GENERIC, message);
+ return;
+ }
+
+ SearchableOxmEntityDescriptor searchableDescriptor = searchableEntityLookup
+ .getSearchableEntityDescriptors().get(txn.getDescriptor().getEntityName());
+
+ try {
+ if (searchableDescriptor.hasSearchableAttributes()) {
+
+ final String jsonResult = txn.getOperationResult().getResult();
+ if (jsonResult != null && jsonResult.length() > 0) {
+
+ SearchableEntity se = new SearchableEntity();
+ se.setLink( txn.getLink() );
+ populateSearchableEntityDocument(se, jsonResult, searchableDescriptor);
+ se.deriveFields();
+
+
+ String link = null;
+ try {
+ link = elasticSearchAdapter.buildElasticSearchGetDocUrl(getIndexName(), se.getId());
+ } catch (Exception exc) {
+ LOG.error(AaiUiMsgs.ES_FAILED_TO_CONSTRUCT_QUERY, exc.getLocalizedMessage());
+ }
+
+ if (link != null) {
+ NetworkTransaction n2 = new NetworkTransaction();
+ n2.setLink(link);
+ n2.setEntityType(txn.getEntityType());
+ n2.setDescriptor(txn.getDescriptor());
+ n2.setOperationType(HttpMethod.GET);
+
+ esWorkOnHand.incrementAndGet();
+
+ supplyAsync(new PerformElasticSearchRetrieval(n2, elasticSearchAdapter), esExecutor)
+ .whenComplete((result, error) -> {
+
+ esWorkOnHand.decrementAndGet();
+
+ if (error != null) {
+ LOG.error(AaiUiMsgs.ES_RETRIEVAL_FAILED, error.getLocalizedMessage());
+ } else {
+ updateElasticSearchCounters(result);
+ performDocumentUpsert(result, se);
+ }
+ });
+ }
+ }
+
+ }
+ } catch (JsonProcessingException exc) {
+ LOG.error(AaiUiMsgs.ERROR_GENERIC, "Processing error while fetching document for elasticsearch update. Error: " + exc.getMessage() );
+ } catch (IOException exc) {
+ LOG.error(AaiUiMsgs.ERROR_GENERIC, "Processing error while fetching document for elasticsearch update. Error: " + exc.getMessage() );
+ }
+ }
+
+ /**
+ * Process store document result.
+ *
+ * @param esPutResult the es put result
+ * @param esGetResult the es get result
+ * @param se the se
+ */
+ private void processStoreDocumentResult(NetworkTransaction esPutResult,
+ NetworkTransaction esGetResult, SearchableEntity se) {
+
+ OperationResult or = esPutResult.getOperationResult();
+
+ if (!or.wasSuccessful()) {
+ if (or.getResultCode() == VERSION_CONFLICT_EXCEPTION_CODE) {
+
+ if (shouldAllowRetry(se.getId())) {
+ esWorkOnHand.incrementAndGet();
+
+ RetrySearchableEntitySyncContainer rsc =
+ new RetrySearchableEntitySyncContainer(esGetResult, se);
+ retryQueue.push(rsc);
+
+ String message = "Store document failed during searchable entity synchronization"
+ + " due to version conflict. Entity will be re-synced.";
+ LOG.warn(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ }
+ } else {
+ String message =
+ "Store document failed during searchable entity synchronization with result code "
+ + or.getResultCode() + " and result message " + or.getResult();
+ LOG.error(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ }
+ }
+ }
+
+ /**
+ * Perform retry sync.
+ */
+ private void performRetrySync() {
+ while (retryQueue.peek() != null) {
+
+ RetrySearchableEntitySyncContainer rsc = retryQueue.poll();
+ if (rsc != null) {
+
+ SearchableEntity se = rsc.getSearchableEntity();
+ NetworkTransaction txn = rsc.getNetworkTransaction();
+
+ String link = null;
+ try {
+ /*
+ * In this retry flow the se object has already derived its fields
+ */
+ link = elasticSearchAdapter.buildElasticSearchGetDocUrl(getIndexName(), se.getId());
+ } catch (Exception exc) {
+ LOG.error(AaiUiMsgs.ES_FAILED_TO_CONSTRUCT_URI, exc.getLocalizedMessage());
+ }
+
+ if (link != null) {
+ NetworkTransaction retryTransaction = new NetworkTransaction();
+ retryTransaction.setLink(link);
+ retryTransaction.setEntityType(txn.getEntityType());
+ retryTransaction.setDescriptor(txn.getDescriptor());
+ retryTransaction.setOperationType(HttpMethod.GET);
+
+ /*
+ * IMPORTANT - DO NOT incrementAndGet the esWorkOnHand as this is a retry flow! We already
+ * called incrementAndGet when queuing the failed PUT!
+ */
+
+ supplyAsync(new PerformElasticSearchRetrieval(retryTransaction, elasticSearchAdapter),
+ esExecutor).whenComplete((result, error) -> {
+
+ esWorkOnHand.decrementAndGet();
+
+ if (error != null) {
+ LOG.error(AaiUiMsgs.ES_RETRIEVAL_FAILED_RESYNC, error.getLocalizedMessage());
+ } else {
+ updateElasticSearchCounters(result);
+ performDocumentUpsert(result, se);
+ }
+ });
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Should allow retry.
+ *
+ * @param id the id
+ * @return true, if successful
+ */
+ private boolean shouldAllowRetry(String id) {
+ boolean isRetryAllowed = true;
+ if (retryLimitTracker.get(id) != null) {
+ Integer currentCount = retryLimitTracker.get(id);
+ if (currentCount.intValue() >= RETRY_COUNT_PER_ENTITY_LIMIT.intValue()) {
+ isRetryAllowed = false;
+ String message = "Searchable entity re-sync limit reached for " + id
+ + ", re-sync will no longer be attempted for this entity";
+ LOG.error(AaiUiMsgs.ES_SEARCHABLE_ENTITY_SYNC_ERROR, message);
+ } else {
+ Integer newCount = new Integer(currentCount.intValue() + 1);
+ retryLimitTracker.put(id, newCount);
+ }
+ } else {
+ Integer firstRetryCount = new Integer(1);
+ retryLimitTracker.put(id, firstRetryCount);
+ }
+
+ return isRetryAllowed;
+ }
+
+ @Override
+ public SynchronizerState getState() {
+ if (!isSyncDone()) {
+ return SynchronizerState.PERFORMING_SYNCHRONIZATION;
+ }
+
+ return SynchronizerState.IDLE;
+
+ }
+
+ /* (non-Javadoc)
+ * @see org.openecomp.sparky.synchronizer.IndexSynchronizer#getStatReport(boolean)
+ */
+ @Override
+ public String getStatReport(boolean showFinalReport) {
+ syncDurationInMs = System.currentTimeMillis() - syncStartedTimeStampInMs;
+ return this.getStatReport(syncDurationInMs, showFinalReport);
+ }
+
+ /* (non-Javadoc)
+ * @see org.openecomp.sparky.synchronizer.IndexSynchronizer#shutdown()
+ */
+ @Override
+ public void shutdown() {
+ this.shutdownExecutors();
+ }
+
+ @Override
+ protected boolean isSyncDone() {
+ int totalWorkOnHand = aaiWorkOnHand.get() + esWorkOnHand.get();
+
+ if (totalWorkOnHand > 0 || !allWorkEnumerated) {
+ return false;
+ }
+
+ return true;
+ }
+
+}
diff --git a/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoSyncController.java b/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoSyncController.java
new file mode 100644
index 0000000..1fdadfc
--- /dev/null
+++ b/src/test/java/org/onap/aai/sparky/viewandinspect/sync/ViewInspectGizmoSyncController.java
@@ -0,0 +1,106 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * Copyright © 2017 Amdocs
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+package org.onap.aai.sparky.viewandinspect.sync;
+
+import org.onap.aai.sparky.config.oxm.OxmEntityContainerLookup;
+import org.onap.aai.sparky.config.oxm.OxmEntityLookup;
+import org.onap.aai.sparky.config.oxm.SearchableEntityLookup;
+import org.onap.aai.sparky.dal.ElasticSearchAdapter;
+import org.onap.aai.sparky.dal.GizmoAdapter;
+import org.onap.aai.sparky.sync.ElasticSearchIndexCleaner;
+import org.onap.aai.sparky.sync.ElasticSearchSchemaFactory;
+import org.onap.aai.sparky.sync.IndexCleaner;
+import org.onap.aai.sparky.sync.IndexIntegrityValidator;
+import org.onap.aai.sparky.sync.SyncControllerImpl;
+import org.onap.aai.sparky.sync.SyncControllerRegistrar;
+import org.onap.aai.sparky.sync.SyncControllerRegistry;
+import org.onap.aai.sparky.sync.config.ElasticSearchEndpointConfig;
+import org.onap.aai.sparky.sync.config.ElasticSearchSchemaConfig;
+import org.onap.aai.sparky.sync.config.NetworkStatisticsConfig;
+import org.onap.aai.sparky.sync.config.SyncControllerConfig;
+
+public class ViewInspectGizmoSyncController extends SyncControllerImpl
+ implements SyncControllerRegistrar {
+
+ private SyncControllerRegistry syncControllerRegistry;
+ //private GizmoAdapter gizmoAdapter;
+ //private ElasticSearchAdapter esAdapter;
+ //private ElasticSearchSchemaConfig schemaConfig;
+ //private ElasticSearchEndpointConfig endpointConfig;
+
+ public ViewInspectGizmoSyncController(SyncControllerConfig syncControllerConfig,
+ GizmoAdapter gizmoAdapter, ElasticSearchAdapter esAdapter,
+ ElasticSearchSchemaConfig schemaConfig, ElasticSearchEndpointConfig endpointConfig,
+ NetworkStatisticsConfig gizmoStatConfig, NetworkStatisticsConfig esStatConfig,
+ OxmEntityLookup oxmEntityLookup,
+ SearchableEntityLookup searchableEntityLookup, OxmEntityContainerLookup oxmEntityContainerLookup) throws Exception {
+ super(syncControllerConfig);
+
+ // final String controllerName = "View and Inspect Entity Synchronizer";
+
+ //this.gizmoAdapter = gizmoAdapter;
+ //this.esAdapter = esAdapter;
+ //this.schemaConfig = schemaConfig;
+ //this.endpointConfig = endpointConfig;
+
+ IndexIntegrityValidator indexValidator = new IndexIntegrityValidator(esAdapter, schemaConfig,
+ endpointConfig, ElasticSearchSchemaFactory.getIndexSchema(schemaConfig));
+
+ registerIndexValidator(indexValidator);
+
+ ViewInspectGizmoEntitySynchronizer ses = new ViewInspectGizmoEntitySynchronizer(schemaConfig,
+ syncControllerConfig.getNumInternalSyncWorkers(),
+ syncControllerConfig.getNumSyncActiveInventoryWorkers(),
+ syncControllerConfig.getNumSyncElasticWorkers(), gizmoStatConfig, esStatConfig,
+ oxmEntityLookup, searchableEntityLookup, oxmEntityContainerLookup);
+
+ ses.setGizmoAdapter(gizmoAdapter);
+ ses.setElasticSearchAdapter(esAdapter);
+
+ registerEntitySynchronizer(ses);
+
+ IndexCleaner indexCleaner =
+ new ElasticSearchIndexCleaner(esAdapter, endpointConfig, schemaConfig);
+
+ registerIndexCleaner(indexCleaner);
+
+ }
+
+ public SyncControllerRegistry getSyncControllerRegistry() {
+ return syncControllerRegistry;
+ }
+
+ public void setSyncControllerRegistry(SyncControllerRegistry syncControllerRegistry) {
+ this.syncControllerRegistry = syncControllerRegistry;
+ }
+
+ @Override
+ public void registerController() {
+ if ( syncControllerRegistry != null ) {
+ if ( syncControllerConfig.isEnabled()) {
+ syncControllerRegistry.registerSyncController(this);
+ }
+ }
+
+ }
+}