Fix cascaded get filtering and speed

This change adds proper cascading bahaviour to the returned service
template for filtered gets on policies.

It also significantly improves the performance on get operations, all
get operations on the API now result in only one database query.

Issue-ID: POLICY-1402
Change-Id: I7c7d0a8d9f1caf8b610461779e242c34b7f77365
Signed-off-by: liamfallon <liam.fallon@est.tech>

3 files changed, 172 insertions, 70 deletions

diff --git a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java
index 311686831..11ecadb53 100644
--- a/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java
+++ b/models-provider/src/test/java/org/onap/policy/models/provider/impl/PolicyToscaPersistenceTest.java
@@ -44,6 +44,7 @@ import org.onap.policy.models.provider.PolicyModelsProviderFactory;
 import org.onap.policy.models.provider.PolicyModelsProviderParameters;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicy;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyFilter;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeFilter;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -108,6 +109,34 @@ public class PolicyToscaPersistenceTest {
     }
 
     @Test
+    public void testHpaPolicyTypeGet() throws PfModelException {
+        long getStartTime = System.currentTimeMillis();
+        ToscaServiceTemplate hpaServiceTemplate =
+                databaseProvider.getPolicyTypes("onap.policies.optimization.resource.HpaPolicy", "1.0.0");
+        LOGGER.trace("HPA policy normal get time (ms): {}", System.currentTimeMillis() - getStartTime);
+
+        assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size());
+        assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size());
+
+        getStartTime = System.currentTimeMillis();
+        ToscaPolicyTypeFilter hpaFilter = ToscaPolicyTypeFilter.builder()
+                .name("onap.policies.optimization.resource.HpaPolicy").version("1.0.0").build();
+        hpaServiceTemplate = databaseProvider.getFilteredPolicyTypes(hpaFilter);
+        LOGGER.trace("HPA policy filter name version get time (ms): {}", System.currentTimeMillis() - getStartTime);
+
+        assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size());
+        assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size());
+
+        getStartTime = System.currentTimeMillis();
+        hpaFilter = ToscaPolicyTypeFilter.builder().name("onap.policies.optimization.resource.HpaPolicy").build();
+        hpaServiceTemplate = databaseProvider.getFilteredPolicyTypes(hpaFilter);
+        LOGGER.trace("HPA policy filter name only get time (ms): {}", System.currentTimeMillis() - getStartTime);
+
+        assertEquals(3, hpaServiceTemplate.getPolicyTypesAsMap().size());
+        assertEquals(5, hpaServiceTemplate.getDataTypesAsMap().size());
+    }
+
+    @Test
     public void testNamingPolicyGet() throws PfModelException {
         String policyYamlString = ResourceUtils.getResourceAsString("policies/sdnc.policy.naming.input.tosca.yaml");
         ToscaServiceTemplate serviceTemplate =
@@ -120,7 +149,26 @@ public class PolicyToscaPersistenceTest {
         long getStartTime = System.currentTimeMillis();
         ToscaServiceTemplate namingServiceTemplate =
                 databaseProvider.getPolicies("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP", "1.0.0");
-        LOGGER.trace("Naming policy get time (ms): {}", System.currentTimeMillis() - getStartTime);
+        LOGGER.trace("Naming policy normal get time (ms): {}", System.currentTimeMillis() - getStartTime);
+
+        assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size());
+        assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size());
+        assertEquals(3, namingServiceTemplate.getDataTypesAsMap().size());
+
+        getStartTime = System.currentTimeMillis();
+        ToscaPolicyFilter filter =
+                ToscaPolicyFilter.builder().name("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP").version("1.0.0").build();
+        namingServiceTemplate = databaseProvider.getFilteredPolicies(filter);
+        LOGGER.trace("Naming policy filtered get time (ms): {}", System.currentTimeMillis() - getStartTime);
+
+        assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size());
+        assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size());
+        assertEquals(3, namingServiceTemplate.getDataTypesAsMap().size());
+
+        getStartTime = System.currentTimeMillis();
+        filter = ToscaPolicyFilter.builder().name("SDNC_Policy.ONAP_VNF_NAMING_TIMESTAMP").build();
+        namingServiceTemplate = databaseProvider.getFilteredPolicies(filter);
+        LOGGER.trace("Naming policy filtered name only get time (ms): {}", System.currentTimeMillis() - getStartTime);
 
         assertEquals(1, namingServiceTemplate.getToscaTopologyTemplate().getPoliciesAsMap().size());
         assertEquals(1, namingServiceTemplate.getPolicyTypesAsMap().size());
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
index a65cdbe69..65cce48db 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/authorative/provider/AuthorativeToscaProvider.java
@@ -23,15 +23,15 @@ package org.onap.policy.models.tosca.authorative.provider;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 
+import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
 import lombok.NonNull;
 
+import org.apache.commons.collections4.CollectionUtils;
 import org.onap.policy.models.base.PfConceptKey;
 import org.onap.policy.models.base.PfModelException;
 import org.onap.policy.models.base.PfModelRuntimeException;
@@ -44,6 +44,7 @@ import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeFilter;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaServiceTemplate;
 import org.onap.policy.models.tosca.simple.concepts.JpaToscaServiceTemplate;
 import org.onap.policy.models.tosca.simple.provider.SimpleToscaProvider;
+import org.onap.policy.models.tosca.utils.ToscaServiceTemplateUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -116,17 +117,33 @@ public class AuthorativeToscaProvider {
             @NonNull final ToscaPolicyTypeFilter filter) throws PfModelException {
 
         LOGGER.debug("->getFilteredPolicyTypes: filter={}", filter);
+        SimpleToscaProvider simpleToscaProvider = new SimpleToscaProvider();
 
-        ToscaServiceTemplate serviceTemplate =
-                new SimpleToscaProvider().getPolicyTypes(dao, null, null).toAuthorative();
+        final JpaToscaServiceTemplate dbServiceTemplate = simpleToscaProvider.getPolicyTypes(dao, null, null);
 
-        List<ToscaPolicyType> filteredPolicyTypes = new ArrayList<>(serviceTemplate.getPolicyTypes().values());
+        List<ToscaPolicyType> filteredPolicyTypes =
+                new ArrayList<>(dbServiceTemplate.toAuthorative().getPolicyTypes().values());
         filteredPolicyTypes = filter.filter(filteredPolicyTypes);
 
-        serviceTemplate.setPolicyTypes(asConceptMap(filteredPolicyTypes));
+        if (CollectionUtils.isEmpty(filteredPolicyTypes)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policy types for filter " + filter.toString() + " do not exist");
+        }
 
-        LOGGER.debug("<-getFilteredPolicyTypes: filter={}, serviceTemplate={}", filter, serviceTemplate);
-        return serviceTemplate;
+        JpaToscaServiceTemplate filteredServiceTemplate = new JpaToscaServiceTemplate();
+
+        for (ToscaPolicyType policyType : filteredPolicyTypes) {
+            JpaToscaServiceTemplate cascadedServiceTemplate = simpleToscaProvider
+                    .getCascadedPolicyTypes(dbServiceTemplate, policyType.getName(), policyType.getVersion());
+
+            filteredServiceTemplate =
+                    ToscaServiceTemplateUtils.addFragment(filteredServiceTemplate, cascadedServiceTemplate);
+        }
+
+        ToscaServiceTemplate returnServiceTemplate = filteredServiceTemplate.toAuthorative();
+
+        LOGGER.debug("<-getFilteredPolicyTypes: filter={}, serviceTemplate={}", filter, returnServiceTemplate);
+        return returnServiceTemplate;
     }
 
     /**
@@ -272,16 +289,33 @@ public class AuthorativeToscaProvider {
         LOGGER.debug("->getFilteredPolicies: filter={}", filter);
         String version = ToscaPolicyFilter.LATEST_VERSION.equals(filter.getVersion()) ? null : filter.getVersion();
 
-        ToscaServiceTemplate serviceTemplate =
-                new SimpleToscaProvider().getPolicies(dao, filter.getName(), version).toAuthorative();
+        SimpleToscaProvider simpleToscaProvider = new SimpleToscaProvider();
+        final JpaToscaServiceTemplate dbServiceTemplate =
+                simpleToscaProvider.getPolicies(dao, filter.getName(), version);
 
-        List<ToscaPolicy> filteredPolicies = asConceptList(serviceTemplate.getToscaTopologyTemplate().getPolicies());
+        List<ToscaPolicy> filteredPolicies =
+                asConceptList(dbServiceTemplate.toAuthorative().getToscaTopologyTemplate().getPolicies());
         filteredPolicies = filter.filter(filteredPolicies);
 
-        serviceTemplate.getToscaTopologyTemplate().setPolicies(asConceptMapList(filteredPolicies));
+        if (CollectionUtils.isEmpty(filteredPolicies)) {
+            throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
+                    "policies for filter " + filter.toString() + " do not exist");
+        }
 
-        LOGGER.debug("<-getFilteredPolicies: filter={}, serviceTemplate={}", filter, serviceTemplate);
-        return serviceTemplate;
+        JpaToscaServiceTemplate filteredServiceTemplate = new JpaToscaServiceTemplate();
+
+        for (ToscaPolicy policy : filteredPolicies) {
+            JpaToscaServiceTemplate cascadedServiceTemplate =
+                    simpleToscaProvider.getCascadedPolicies(dbServiceTemplate, policy.getName(), policy.getVersion());
+
+            filteredServiceTemplate =
+                    ToscaServiceTemplateUtils.addFragment(filteredServiceTemplate, cascadedServiceTemplate);
+        }
+
+        ToscaServiceTemplate returnServiceTemplate = filteredServiceTemplate.toAuthorative();
+
+        LOGGER.debug("<-getFilteredPolicies: filter={}, serviceTemplate={}", filter, returnServiceTemplate);
+        return returnServiceTemplate;
     }
 
     /**
@@ -384,38 +418,6 @@ public class AuthorativeToscaProvider {
     }
 
     /**
-     * Return the contents of a list of concepts as a list of maps of concepts.
-     *
-     * @param conceptList the concept list
-     * @return the list of concept map
-     */
-    private <T extends ToscaEntity> List<Map<String, T>> asConceptMapList(List<T> conceptList) {
-        List<Map<String, T>> toscaEntityMapList = new ArrayList<>();
-        for (T concept : conceptList) {
-            Map<String, T> conceptMap = new TreeMap<>();
-            conceptMap.put(concept.getName(), concept);
-            toscaEntityMapList.add(conceptMap);
-        }
-
-        return toscaEntityMapList;
-    }
-
-    /**
-     * Return the contents of a list of concepts as map of concepts.
-     *
-     * @param conceptList the concept list
-     * @return the list of concept map
-     */
-    private <T extends ToscaEntity> Map<String, T> asConceptMap(List<T> conceptList) {
-        Map<String, T> conceptMap = new LinkedHashMap<>();
-        for (T concept : conceptList) {
-            conceptMap.put(concept.getName(), concept);
-        }
-
-        return conceptMap;
-    }
-
-    /**
      * Handle a PfModelRuntimeException on a list call.
      *
      * @param pfme the model exception
diff --git a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
index a3e18ca41..805f9dfed 100644
--- a/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
+++ b/models-tosca/src/main/java/org/onap/policy/models/tosca/simple/provider/SimpleToscaProvider.java
@@ -133,13 +133,32 @@ public class SimpleToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getDataTypes: name={}, version={}", name, version);
 
-        JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao);
+        final JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao);
 
-        if (!ToscaUtils.doDataTypesExist(serviceTemplate)) {
+        if (!ToscaUtils.doDataTypesExist(dbServiceTemplate)) {
             throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
                     "data types for " + name + ":" + version + DO_NOT_EXIST);
         }
 
+        JpaToscaServiceTemplate serviceTemplate = getCascadedDataTypes(dbServiceTemplate, name, version);
+
+        LOGGER.debug("<-getDataTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
+        return serviceTemplate;
+    }
+
+    /**
+     * Get the cascaded data types for a data type name and version.
+     *
+     * @param dbServiceTemplate the service template to search for the cascaded data types
+     * @param name the data type name we are searching for
+     * @param version the data type version we are searching for
+     * @return a service template containing the cascaded data types
+     * @throws PfModelException on errors getting the data types
+     */
+    public JpaToscaServiceTemplate getCascadedDataTypes(@NonNull final JpaToscaServiceTemplate dbServiceTemplate,
+            final String name, final String version) throws PfModelException {
+
+        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate);
         serviceTemplate.setPolicyTypes(null);
         serviceTemplate.setTopologyTemplate(null);
 
@@ -154,15 +173,13 @@ public class SimpleToscaProvider {
             Collection<PfConceptKey> referencedDataTypeKeys = dataType.getReferencedDataTypes();
 
             for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) {
-                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate =
-                        getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
+                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = getCascadedDataTypes(dbServiceTemplate,
+                        referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
 
                 serviceTemplate =
                         ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeEntityTreeServiceTemplate);
             }
         }
-
-        LOGGER.debug("<-getDataTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
         return serviceTemplate;
     }
 
@@ -277,16 +294,36 @@ public class SimpleToscaProvider {
             throws PfModelException {
         LOGGER.debug("->getPolicyTypes: name={}, version={}", name, version);
 
-        JpaToscaServiceTemplate serviceTemplate = getServiceTemplate(dao);
-
-        serviceTemplate.setDataTypes(null);
-        serviceTemplate.setTopologyTemplate(null);
+        final JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao);
 
-        if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) {
+        if (!ToscaUtils.doPolicyTypesExist(dbServiceTemplate)) {
             throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
                     "policy types for " + name + ":" + version + DO_NOT_EXIST);
         }
 
+        JpaToscaServiceTemplate serviceTemplate = getCascadedPolicyTypes(dbServiceTemplate, name, version);
+
+        LOGGER.debug("<-getPolicyTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
+        return serviceTemplate;
+    }
+
+    /**
+     * Get the cascaded policy types for a policy type name and version.
+     *
+     * @param dbServiceTemplate the service template to search for the cascaded policy types
+     * @param name the policy type name we are searching for
+     * @param version the policy type version we are searching for
+     * @return a service template containing the cascaded policy types
+     * @throws PfModelException on errors getting the policy types
+     */
+    public JpaToscaServiceTemplate getCascadedPolicyTypes(final JpaToscaServiceTemplate dbServiceTemplate,
+            final String name, final String version) throws PfModelException {
+
+        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate);
+
+        serviceTemplate.setDataTypes(null);
+        serviceTemplate.setTopologyTemplate(null);
+
         ToscaUtils.getEntityTree(serviceTemplate.getPolicyTypes(), name, version);
 
         if (!ToscaUtils.doPolicyTypesExist(serviceTemplate)) {
@@ -301,8 +338,8 @@ public class SimpleToscaProvider {
             Collection<PfConceptKey> referencedDataTypeKeys = policyType.getReferencedDataTypes();
 
             for (PfConceptKey referencedDataTypeKey : referencedDataTypeKeys) {
-                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate =
-                        getDataTypes(dao, referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
+                JpaToscaServiceTemplate dataTypeEntityTreeServiceTemplate = getCascadedDataTypes(dbServiceTemplate,
+                        referencedDataTypeKey.getName(), referencedDataTypeKey.getVersion());
 
                 dataTypeServiceTemplate = ToscaServiceTemplateUtils.addFragment(dataTypeServiceTemplate,
                         dataTypeEntityTreeServiceTemplate);
@@ -310,8 +347,6 @@ public class SimpleToscaProvider {
         }
 
         serviceTemplate = ToscaServiceTemplateUtils.addFragment(serviceTemplate, dataTypeServiceTemplate);
-
-        LOGGER.debug("<-getPolicyTypes: name={}, version={}, serviceTemplate={}", name, version, serviceTemplate);
         return serviceTemplate;
     }
 
@@ -438,15 +473,33 @@ public class SimpleToscaProvider {
 
         JpaToscaServiceTemplate dbServiceTemplate = getServiceTemplate(dao);
 
-        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate);
-        serviceTemplate.setDataTypes(new JpaToscaDataTypes());
-        serviceTemplate.setPolicyTypes(new JpaToscaPolicyTypes());
-
-        if (!ToscaUtils.doPoliciesExist(serviceTemplate)) {
+        if (!ToscaUtils.doPoliciesExist(dbServiceTemplate)) {
             throw new PfModelRuntimeException(Response.Status.NOT_FOUND,
                     "policies for " + name + ":" + version + DO_NOT_EXIST);
         }
 
+        JpaToscaServiceTemplate returnServiceTemplate = getCascadedPolicies(dbServiceTemplate, name, version);
+
+        LOGGER.debug("<-getPolicies: name={}, version={}, serviceTemplate={}", name, version, returnServiceTemplate);
+        return returnServiceTemplate;
+    }
+
+    /**
+     * Get the cascaded policies for a policy name and version.
+     *
+     * @param dbServiceTemplate the service template to search for the cascaded policy
+     * @param name the policy name we are searching for
+     * @param version the policy version we are searching for
+     * @return a service template containing the cascaded policy
+     * @throws PfModelException on errors getting the policy
+     */
+    public JpaToscaServiceTemplate getCascadedPolicies(final JpaToscaServiceTemplate dbServiceTemplate,
+            final String name, final String version) throws PfModelException {
+
+        JpaToscaServiceTemplate serviceTemplate = new JpaToscaServiceTemplate(dbServiceTemplate);
+        serviceTemplate.setDataTypes(new JpaToscaDataTypes());
+        serviceTemplate.setPolicyTypes(new JpaToscaPolicyTypes());
+
         ToscaUtils.getEntityTree(serviceTemplate.getTopologyTemplate().getPolicies(), name, version);
 
         if (!ToscaUtils.doPoliciesExist(serviceTemplate)) {
@@ -458,15 +511,14 @@ public class SimpleToscaProvider {
         returnServiceTemplate.getTopologyTemplate().setPolicies(new JpaToscaPolicies());
 
         for (JpaToscaPolicy policy : serviceTemplate.getTopologyTemplate().getPolicies().getConceptMap().values()) {
-            JpaToscaServiceTemplate referencedEntitiesServiceTemplate =
-                    getPolicyTypes(dao, policy.getType().getName(), policy.getType().getVersion());
+            JpaToscaServiceTemplate referencedEntitiesServiceTemplate = getCascadedPolicyTypes(dbServiceTemplate,
+                    policy.getType().getName(), policy.getType().getVersion());
 
             returnServiceTemplate.getTopologyTemplate().getPolicies().getConceptMap().put(policy.getKey(), policy);
             returnServiceTemplate =
                     ToscaServiceTemplateUtils.addFragment(returnServiceTemplate, referencedEntitiesServiceTemplate);
         }
 
-        LOGGER.debug("<-getPolicies: name={}, version={}, serviceTemplate={}", name, version, returnServiceTemplate);
         return returnServiceTemplate;
     }