Fix performance degradation bug

- do NOT use cpsPath query for gettign individual CM Handles
- change order of queries for more efficency (property query first)

Issue-ID: CPS-1085
Change-Id: Ia3bdf4f0d245a2b128557352685d2aa17c1f939c
Signed-off-by: kissand <andras.zoltan.kiss@est.tech>
Signed-off-by: ToineSiebelink <toine.siebelink@est.tech>

3 files changed, 142 insertions, 88 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NetworkCmProxyCmHandlerQueryService.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NetworkCmProxyCmHandlerQueryService.java
index 92b1e82c3e..faf58b95bf 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NetworkCmProxyCmHandlerQueryService.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/NetworkCmProxyCmHandlerQueryService.java
@@ -20,9 +20,9 @@
 
 package org.onap.cps.ncmp.api;
 
-import java.util.Collection;
+import java.util.Set;
+import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
 import org.onap.cps.spi.model.CmHandleQueryServiceParameters;
-import org.onap.cps.spi.model.DataNode;
 
 public interface NetworkCmProxyCmHandlerQueryService {
     /**
@@ -31,5 +31,13 @@ public interface NetworkCmProxyCmHandlerQueryService {
      * @param cmHandleQueryServiceParameters the cm handle query parameters
      * @return collection of cm handles
      */
-    Collection<DataNode> queryCmHandles(CmHandleQueryServiceParameters cmHandleQueryServiceParameters);
+    Set<NcmpServiceCmHandle> queryCmHandles(CmHandleQueryServiceParameters cmHandleQueryServiceParameters);
+
+    /**
+     * Query and return cm handles that match the given query parameters.
+     *
+     * @param cmHandleQueryServiceParameters the cm handle query parameters
+     * @return collection of cm handle ids
+     */
+    Set<String> queryCmHandleIds(CmHandleQueryServiceParameters cmHandleQueryServiceParameters);
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyCmHandlerQueryServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyCmHandlerQueryServiceImpl.java
index 3deaa7d237..d64dfb4a63 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyCmHandlerQueryServiceImpl.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyCmHandlerQueryServiceImpl.java
@@ -22,6 +22,7 @@ package org.onap.cps.ncmp.api.impl;
 
 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NCMP_DATASPACE_NAME;
 import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NCMP_DMI_REGISTRY_ANCHOR;
+import static org.onap.cps.ncmp.api.impl.utils.YangDataConverter.convertYangModelCmHandleToNcmpServiceCmHandle;
 import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS;
 import static org.onap.cps.utils.CmHandleQueryRestParametersValidator.validateModuleNameConditionProperties;
 
@@ -29,20 +30,22 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandlerQueryService;
+import org.onap.cps.ncmp.api.impl.utils.YangDataConverter;
+import org.onap.cps.ncmp.api.models.NcmpServiceCmHandle;
 import org.onap.cps.spi.CpsAdminPersistenceService;
 import org.onap.cps.spi.CpsDataPersistenceService;
 import org.onap.cps.spi.model.Anchor;
 import org.onap.cps.spi.model.CmHandleQueryServiceParameters;
 import org.onap.cps.spi.model.ConditionProperties;
 import org.onap.cps.spi.model.DataNode;
-import org.onap.cps.spi.model.DataNodeIdentifier;
-import org.onap.cps.utils.JsonObjectMapper;
 import org.springframework.stereotype.Service;
 
 @Service
@@ -52,9 +55,9 @@ public class NetworkCmProxyCmHandlerQueryServiceImpl implements NetworkCmProxyCm
 
     private static final String PROPERTY_QUERY_NAME = "hasAllProperties";
     private static final String MODULE_QUERY_NAME = "hasAllModules";
+    private static final Object NO_QUERY_EXECUTED = null;
     private final CpsDataPersistenceService cpsDataPersistenceService;
     private final CpsAdminPersistenceService cpsAdminPersistenceService;
-    private final JsonObjectMapper jsonObjectMapper;
 
     /**
      * Query and return cm handles that match the given query parameters.
@@ -63,86 +66,124 @@ public class NetworkCmProxyCmHandlerQueryServiceImpl implements NetworkCmProxyCm
      * @return collection of cm handles
      */
     @Override
-    public Collection<DataNode> queryCmHandles(final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
+    public Set<NcmpServiceCmHandle> queryCmHandles(
+        final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
 
         if (cmHandleQueryServiceParameters.getCmHandleQueryParameters().isEmpty()) {
             return getAllCmHandles();
         }
 
-        final Collection<DataNodeIdentifier> amalgamatedQueryResultIdentifiers = new ArrayList<>();
-        final Map<DataNodeIdentifier, DataNode> amalgamatedQueryResults = new HashMap<>();
+        final Map<String, NcmpServiceCmHandle> publicPropertyQueryResult
+            = executePublicPropertyQueries(cmHandleQueryServiceParameters);
 
-        final boolean firstQuery = moduleNameQuery(cmHandleQueryServiceParameters,
-                amalgamatedQueryResultIdentifiers, amalgamatedQueryResults);
+        final Map<String, NcmpServiceCmHandle> combinedQueryResult =
+            combineWithModuleNameQuery(cmHandleQueryServiceParameters, publicPropertyQueryResult);
 
-        publicPropertyQuery(cmHandleQueryServiceParameters, amalgamatedQueryResultIdentifiers,
-                amalgamatedQueryResults, firstQuery);
+        return combinedQueryResult == NO_QUERY_EXECUTED
+            ? Collections.emptySet() : new HashSet<NcmpServiceCmHandle>(combinedQueryResult.values());
+    }
 
-        final Collection<DataNode> filteredDataNodes = new ArrayList<>();
-        amalgamatedQueryResultIdentifiers.forEach(amalgamatedQueryResultIdentifier ->
-            filteredDataNodes.add(amalgamatedQueryResults.get(amalgamatedQueryResultIdentifier))
-        );
+    /**
+     * Query and return cm handles that match the given query parameters.
+     *
+     * @param cmHandleQueryServiceParameters the cm handle query parameters
+     * @return collection of cm handle ids
+     */
+    @Override
+    public Set<String> queryCmHandleIds(
+        final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
+
+        if (cmHandleQueryServiceParameters.getCmHandleQueryParameters().isEmpty()) {
+            return getAllCmHandleIds();
+        }
+
+        final Map<String, NcmpServiceCmHandle> publicPropertyQueryResult
+            = executePublicPropertyQueries(cmHandleQueryServiceParameters);
+
+        final Collection<String> moduleNamesForQuery =
+            getModuleNamesForQuery(cmHandleQueryServiceParameters.getCmHandleQueryParameters());
+        if (moduleNamesForQuery.isEmpty()) {
+            return publicPropertyQueryResult == NO_QUERY_EXECUTED
+                ? Collections.emptySet() : publicPropertyQueryResult.keySet();
+        }
+        final Set<String> moduleNameQueryResult = getNamesOfAnchorsWithGivenModules(moduleNamesForQuery);
+
+        if (publicPropertyQueryResult == NO_QUERY_EXECUTED) {
+            return moduleNameQueryResult;
+        }
 
-        return filteredDataNodes;
+        moduleNameQueryResult.retainAll(publicPropertyQueryResult.keySet());
+        return moduleNameQueryResult;
     }
 
-    private void publicPropertyQuery(final CmHandleQueryServiceParameters cmHandleQueryServiceParameters,
-                                     final Collection<DataNodeIdentifier> amalgamatedQueryResultIdentifiers,
-                                     final Map<DataNodeIdentifier, DataNode> amalgamatedQueryResults,
-                                     boolean firstQuery) {
-        for (final Map.Entry<String, String> entry :
-                getPublicPropertyPairs(cmHandleQueryServiceParameters.getCmHandleQueryParameters()).entrySet()) {
-            final String cmHandlePath = "//public-properties[@name='" + entry.getKey() + "' " + "and @value='"
-                    + entry.getValue() + "']" + "/ancestor::cm-handles";
-
-            final Collection<DataNode> dataNodes = getDataNodes(cmHandlePath);
-
-            if (firstQuery) {
-                firstQuery = false;
-                dataNodes.forEach(dataNode -> {
-                    final DataNodeIdentifier dataNodeIdentifier =
-                            jsonObjectMapper.convertToValueType(dataNode, DataNodeIdentifier.class);
-                    amalgamatedQueryResultIdentifiers.add(dataNodeIdentifier);
-                    amalgamatedQueryResults.put(dataNodeIdentifier, dataNode);
-                });
+    private Map<String, NcmpServiceCmHandle> executePublicPropertyQueries(
+        final CmHandleQueryServiceParameters cmHandleQueryServiceParameters) {
+        final Map<String, String> publicPropertyQueryPairs =
+            getPublicPropertyPairs(cmHandleQueryServiceParameters.getCmHandleQueryParameters());
+        if (publicPropertyQueryPairs.isEmpty()) {
+            return null;
+        }
+        Map<String, NcmpServiceCmHandle> cmHandleIdToNcmpServiceCmHandles = null;
+        for (final Map.Entry<String, String> entry : publicPropertyQueryPairs.entrySet()) {
+            final String cmHandlePath = "//public-properties[@name='" + entry.getKey() + "' and @value='"
+                + entry.getValue() + "']/ancestor::cm-handles";
+
+            final Collection<DataNode> dataNodes = queryDataNodes(cmHandlePath);
+            if (cmHandleIdToNcmpServiceCmHandles == NO_QUERY_EXECUTED) {
+                cmHandleIdToNcmpServiceCmHandles = collectDataNodesToNcmpServiceCmHandles(dataNodes);
             } else {
-                final Collection<DataNodeIdentifier> singleConditionQueryDataNodeIdentifiers = new ArrayList<>();
-                dataNodes.forEach(dataNode -> {
-                    final DataNodeIdentifier dataNodeIdentifier =
-                            jsonObjectMapper.convertToValueType(dataNode, DataNodeIdentifier.class);
-                    singleConditionQueryDataNodeIdentifiers.add(dataNodeIdentifier);
-                    amalgamatedQueryResults.put(dataNodeIdentifier, dataNode);
-                });
-                amalgamatedQueryResultIdentifiers.retainAll(singleConditionQueryDataNodeIdentifiers);
+                final Collection<String> cmHandleIdsToRetain = dataNodes.parallelStream()
+                    .map(dataNode -> dataNode.getLeaves().get("id").toString()).collect(Collectors.toSet());
+                cmHandleIdToNcmpServiceCmHandles.keySet().retainAll(cmHandleIdsToRetain);
             }
-
-            if (amalgamatedQueryResultIdentifiers.isEmpty()) {
+            if (cmHandleIdToNcmpServiceCmHandles.isEmpty()) {
                 break;
             }
         }
+        return cmHandleIdToNcmpServiceCmHandles;
     }
 
-    private boolean moduleNameQuery(final CmHandleQueryServiceParameters cmHandleQueryServiceParameters,
-                                    final Collection<DataNodeIdentifier> amalgamatedQueryResultIdentifiers,
-                                    final Map<DataNodeIdentifier, DataNode> amalgamatedQueryResults) {
-        boolean firstQuery = true;
-        if (!getModuleNames(cmHandleQueryServiceParameters.getCmHandleQueryParameters()).isEmpty()) {
-            final Collection<String> anchorNames = cpsAdminPersistenceService.queryAnchors("NFP-Operational",
-                    getModuleNames(cmHandleQueryServiceParameters.getCmHandleQueryParameters()))
-                    .parallelStream().map(Anchor::getName).collect(Collectors.toList());
-
-            getAllCmHandles().forEach(dataNode -> {
-                if (anchorNames.contains(dataNode.getLeaves().get("id").toString())) {
-                    final DataNodeIdentifier dataNodeIdentifier =
-                            jsonObjectMapper.convertToValueType(dataNode, DataNodeIdentifier.class);
-                    amalgamatedQueryResultIdentifiers.add(dataNodeIdentifier);
-                    amalgamatedQueryResults.put(dataNodeIdentifier, dataNode);
+    private Map<String, NcmpServiceCmHandle> combineWithModuleNameQuery(
+        final CmHandleQueryServiceParameters cmHandleQueryServiceParameters,
+        final Map<String, NcmpServiceCmHandle> previousQueryResult) {
+        final Collection<String> moduleNamesForQuery =
+            getModuleNamesForQuery(cmHandleQueryServiceParameters.getCmHandleQueryParameters());
+        if (moduleNamesForQuery.isEmpty()) {
+            return previousQueryResult;
+        }
+        final Collection<String> cmHandleIdsByModuleName = getNamesOfAnchorsWithGivenModules(moduleNamesForQuery);
+        if (cmHandleIdsByModuleName.isEmpty()) {
+            return Collections.emptyMap();
+        }
+        final Map<String, NcmpServiceCmHandle> queryResult = new HashMap<>(cmHandleIdsByModuleName.size());
+        if (previousQueryResult == NO_QUERY_EXECUTED) {
+            //TODO Discuss performance/scaling of getting ALL cmHandles here
+            getAllCmHandles().forEach(ncmpServiceCmHandle -> {
+                if (cmHandleIdsByModuleName.contains(ncmpServiceCmHandle.getCmHandleId())) {
+                    queryResult.put(ncmpServiceCmHandle.getCmHandleId(), ncmpServiceCmHandle);
                 }
             });
-
-            firstQuery = false;
+            return queryResult;
         }
-        return firstQuery;
+        previousQueryResult.keySet().retainAll(cmHandleIdsByModuleName);
+        queryResult.putAll(previousQueryResult);
+        return queryResult;
+    }
+
+    private Set<String> getNamesOfAnchorsWithGivenModules(final Collection<String> moduleNamesForQuery) {
+        final Collection<Anchor> anchors =
+            cpsAdminPersistenceService.queryAnchors("NFP-Operational", moduleNamesForQuery);
+        return anchors.parallelStream().map(Anchor::getName).collect(Collectors.toSet());
+    }
+
+    private Map<String, NcmpServiceCmHandle> collectDataNodesToNcmpServiceCmHandles(
+        final Collection<DataNode> dataNodes) {
+        final Map<String, NcmpServiceCmHandle> cmHandleIdToNcmpServiceCmHandle = new HashMap<>();
+        dataNodes.forEach(dataNode -> {
+            final NcmpServiceCmHandle ncmpServiceCmHandle = createNcmpServiceCmHandle(dataNode);
+            cmHandleIdToNcmpServiceCmHandle.put(ncmpServiceCmHandle.getCmHandleId(), ncmpServiceCmHandle);
+        });
+        return cmHandleIdToNcmpServiceCmHandle;
     }
 
     private List<Map<String, String>> getConditions(final List<ConditionProperties> conditionProperties,
@@ -155,13 +196,13 @@ public class NetworkCmProxyCmHandlerQueryServiceImpl implements NetworkCmProxyCm
         return Collections.emptyList();
     }
 
-    private List<String> getModuleNames(final List<ConditionProperties> conditionProperties) {
+    private Collection<String> getModuleNamesForQuery(final List<ConditionProperties> conditionProperties) {
         final List<String> result = new ArrayList<>();
         getConditions(conditionProperties, MODULE_QUERY_NAME).parallelStream().forEach(
-                conditionProperty -> {
-                    validateModuleNameConditionProperties(conditionProperty);
-                    result.add(conditionProperty.get("moduleName"));
-                }
+            conditionProperty -> {
+                validateModuleNameConditionProperties(conditionProperty);
+                result.add(conditionProperty.get("moduleName"));
+            }
         );
         return result;
     }
@@ -172,12 +213,23 @@ public class NetworkCmProxyCmHandlerQueryServiceImpl implements NetworkCmProxyCm
         return result;
     }
 
-    private Collection<DataNode> getAllCmHandles() {
-        return getDataNodes("//public-properties/ancestor::cm-handles");
+    private Set<NcmpServiceCmHandle> getAllCmHandles() {
+        return queryDataNodes("/dmi-registry/cm-handles").stream()
+            .map(this::createNcmpServiceCmHandle).collect(Collectors.toSet());
+    }
+
+    private Set<String> getAllCmHandleIds() {
+        return cpsAdminPersistenceService.getAnchors("NFP-Operational")
+            .parallelStream().map(Anchor::getName).collect(Collectors.toSet());
+    }
+
+    private List<DataNode> queryDataNodes(final String cmHandlePath) {
+        return cpsDataPersistenceService.queryDataNodes(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR,
+            cmHandlePath, INCLUDE_ALL_DESCENDANTS);
     }
 
-    private List<DataNode> getDataNodes(final String cmHandlePath) {
-        return cpsDataPersistenceService.queryDataNodes(
-                NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, cmHandlePath, INCLUDE_ALL_DESCENDANTS);
+    private NcmpServiceCmHandle createNcmpServiceCmHandle(final DataNode dataNode) {
+        return convertYangModelCmHandleToNcmpServiceCmHandle(YangDataConverter
+            .convertCmHandleToYangModel(dataNode, dataNode.getLeaves().get("id").toString()));
     }
 }
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
index bfc740786e..e118bf1e58 100755
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java
@@ -41,7 +41,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.onap.cps.api.CpsAdminService;
 import org.onap.cps.api.CpsDataService;
 import org.onap.cps.api.CpsModuleService;
 import org.onap.cps.ncmp.api.NetworkCmProxyCmHandlerQueryService;
@@ -53,7 +52,6 @@ import org.onap.cps.ncmp.api.impl.yangmodels.YangModelCmHandle;
 import org.onap.cps.ncmp.api.inventory.CmHandleState;
 import org.onap.cps.ncmp.api.inventory.CompositeState;
 import org.onap.cps.ncmp.api.inventory.InventoryPersistence;
-import org.onap.cps.ncmp.api.inventory.sync.ModuleSyncService;
 import org.onap.cps.ncmp.api.models.CmHandleQueryApiParameters;
 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse;
 import org.onap.cps.ncmp.api.models.CmHandleRegistrationResponse.RegistrationError;
@@ -84,14 +82,10 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
 
     private final CpsModuleService cpsModuleService;
 
-    private final CpsAdminService cpsAdminService;
-
     private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler;
 
     private final InventoryPersistence inventoryPersistence;
 
-    private final ModuleSyncService moduleSyncService;
-
     private final NetworkCmProxyCmHandlerQueryService networkCmProxyCmHandlerQueryService;
 
     @Override
@@ -167,16 +161,12 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
      */
     @Override
     public Set<NcmpServiceCmHandle> executeCmHandleSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
-
         final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
                 cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
 
         validateCmHandleQueryParameters(cmHandleQueryServiceParameters);
 
-        return networkCmProxyCmHandlerQueryService.queryCmHandles(cmHandleQueryServiceParameters).stream()
-                .map(dataNode -> YangDataConverter
-                        .convertCmHandleToYangModel(dataNode, dataNode.getLeaves().get("id").toString()))
-                .map(YangDataConverter::convertYangModelCmHandleToNcmpServiceCmHandle).collect(Collectors.toSet());
+        return networkCmProxyCmHandlerQueryService.queryCmHandles(cmHandleQueryServiceParameters);
     }
 
     /**
@@ -187,8 +177,12 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService
      */
     @Override
     public Set<String> executeCmHandleIdSearch(final CmHandleQueryApiParameters cmHandleQueryApiParameters) {
-        return executeCmHandleSearch(cmHandleQueryApiParameters).stream().map(NcmpServiceCmHandle::getCmHandleId)
-                .collect(Collectors.toSet());
+        final CmHandleQueryServiceParameters cmHandleQueryServiceParameters = jsonObjectMapper.convertToValueType(
+                cmHandleQueryApiParameters, CmHandleQueryServiceParameters.class);
+
+        validateCmHandleQueryParameters(cmHandleQueryServiceParameters);
+
+        return networkCmProxyCmHandlerQueryService.queryCmHandleIds(cmHandleQueryServiceParameters);
     }
 
     /**