Policy list and Pdpgroup merger code

Draft code of the policy list/pdpgroup merger + tests, this code will most likely be called by a camel Rest route and the answer will be returned to the client/ui. The policies listing is enriched with new fields pdp group info, like:
  1. In which Pdp group/Subgroup the policy is currently deployed
  2. What are the potential pdpgroups/subgroups where the policy can be deployed

  These info will be shown in the policy clamp UI to the user.

Also some refactoring has been done for the sake of clarity.

Issue-ID: POLICY-2924
Signed-off-by: sebdet <sebastien.determe@intl.att.com>
Change-Id: Iba1ab6fea90735551d7d2e2bbff9ab2efdee093e
Signed-off-by: sebdet <sebastien.determe@intl.att.com>

7 files changed, 177 insertions, 44 deletions

diff --git a/src/main/java/org/onap/policy/clamp/loop/CsarInstaller.java b/src/main/java/org/onap/policy/clamp/loop/CsarInstaller.java
index 753789a68..f46f4227b 100644
--- a/src/main/java/org/onap/policy/clamp/loop/CsarInstaller.java
+++ b/src/main/java/org/onap/policy/clamp/loop/CsarInstaller.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP CLAMP
  * ================================================================================
- * Copyright (C) 2019 AT&T Intellectual Property. All rights
+ * Copyright (C) 2019, 2021 AT&T Intellectual Property. All rights
  *                             reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -31,7 +31,6 @@ import java.util.List;
 import java.util.Map.Entry;
 import org.json.simple.parser.ParseException;
 import org.onap.policy.clamp.clds.client.DcaeInventoryServices;
-import org.onap.policy.clamp.clds.client.PolicyEngineServices;
 import org.onap.policy.clamp.clds.exception.sdc.controller.BlueprintParserException;
 import org.onap.policy.clamp.clds.exception.sdc.controller.SdcArtifactInstallerException;
 import org.onap.policy.clamp.clds.model.dcae.DcaeInventoryResponse;
@@ -48,6 +47,7 @@ import org.onap.policy.clamp.loop.template.LoopTemplate;
 import org.onap.policy.clamp.loop.template.LoopTemplatesRepository;
 import org.onap.policy.clamp.loop.template.PolicyModel;
 import org.onap.policy.clamp.loop.template.PolicyModelsRepository;
+import org.onap.policy.clamp.policy.PolicyEngineServices;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.stereotype.Component;
diff --git a/src/main/java/org/onap/policy/clamp/loop/template/PolicyModelsService.java b/src/main/java/org/onap/policy/clamp/loop/template/PolicyModelsService.java
index f0830360e..c4182bd2f 100644
--- a/src/main/java/org/onap/policy/clamp/loop/template/PolicyModelsService.java
+++ b/src/main/java/org/onap/policy/clamp/loop/template/PolicyModelsService.java
@@ -23,7 +23,6 @@
 
 package org.onap.policy.clamp.loop.template;
 
-import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -32,7 +31,6 @@ import org.onap.policy.clamp.clds.tosca.ToscaSchemaConstants;
 import org.onap.policy.clamp.clds.tosca.ToscaYamlToJsonConvertor;
 import org.onap.policy.clamp.policy.pdpgroup.PdpGroupsAnalyzer;
 import org.onap.policy.clamp.util.SemanticVersioning;
-import org.onap.policy.models.pdp.concepts.PdpGroup;
 import org.onap.policy.models.pdp.concepts.PdpGroups;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -157,7 +155,7 @@ public class PolicyModelsService {
      */
     public void updatePdpGroupInfo(PdpGroups pdpGroups) {
         List<PolicyModel> policyModelsList = policyModelsRepository.findAll();
-        PdpGroupsAnalyzer.updatePdpGroup(policyModelsList, pdpGroups);
+        PdpGroupsAnalyzer.updatePdpGroupOfPolicyModels(policyModelsList, pdpGroups);
         this.policyModelsRepository.saveAll(policyModelsList);
     }
 }
diff --git a/src/main/java/org/onap/policy/clamp/clds/client/PolicyEngineServices.java b/src/main/java/org/onap/policy/clamp/policy/PolicyEngineServices.java
index 46c55ea45..196642589 100644
--- a/src/main/java/org/onap/policy/clamp/clds/client/PolicyEngineServices.java
+++ b/src/main/java/org/onap/policy/clamp/policy/PolicyEngineServices.java
@@ -21,15 +21,11 @@
  *
  */
 
-package org.onap.policy.clamp.clds.client;
+package org.onap.policy.clamp.policy;
 
 import com.att.eelf.configuration.EELFLogger;
 import com.att.eelf.configuration.EELFManager;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
 import java.util.LinkedHashMap;
-import java.util.LinkedList;
-import java.util.List;
 import java.util.Map;
 import org.apache.camel.CamelContext;
 import org.apache.camel.Exchange;
@@ -39,7 +35,6 @@ import org.onap.policy.clamp.clds.sdc.controller.installer.BlueprintMicroService
 import org.onap.policy.clamp.clds.util.JsonUtils;
 import org.onap.policy.clamp.loop.template.PolicyModel;
 import org.onap.policy.clamp.loop.template.PolicyModelsService;
-import org.onap.policy.models.pdp.concepts.PdpGroup;
 import org.onap.policy.models.pdp.concepts.PdpGroups;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
@@ -98,7 +93,7 @@ public class PolicyEngineServices {
     public PolicyModel createPolicyModelFromPolicyEngine(String policyType, String policyVersion) {
         PolicyModel policyModelFound = policyModelsService.getPolicyModel(policyType, policyVersion);
         if (policyModelFound == null) {
-            String policyTosca = this.downloadOnePolicy(policyType, policyVersion);
+            String policyTosca = this.downloadOnePolicyToscaModel(policyType, policyVersion);
             if (policyTosca != null && !policyTosca.isEmpty()) {
                 return policyModelsService.savePolicyModelInNewTransaction(
                         new PolicyModel(policyType, policyTosca, policyVersion));
@@ -130,7 +125,7 @@ public class PolicyEngineServices {
      */
     public void synchronizeAllPolicies() {
         LinkedHashMap<String, Object> loadedYaml;
-        loadedYaml = new Yaml().load(downloadAllPolicies());
+        loadedYaml = new Yaml().load(downloadAllPolicyModels());
         if (loadedYaml == null || loadedYaml.isEmpty()) {
             logger.warn("getAllPolicyType yaml returned by policy engine could not be decoded, as it's null or empty");
             return;
@@ -149,9 +144,9 @@ public class PolicyEngineServices {
      *
      * @return A yaml containing all policy Types and all data types
      */
-    public String downloadAllPolicies() {
+    public String downloadAllPolicyModels() {
         return callCamelRoute(ExchangeBuilder.anExchange(camelContext).build(), "direct:get-all-policy-models",
-                "Get all policies");
+                "Get all policies models");
     }
 
     /**
@@ -161,8 +156,8 @@ public class PolicyEngineServices {
      * @param policyVersion The policy version
      * @return A string with the whole policy tosca model
      */
-    public String downloadOnePolicy(String policyType, String policyVersion) {
-        logger.info("Downloading the policy model " + policyType + "/" + policyVersion);
+    public String downloadOnePolicyToscaModel(String policyType, String policyVersion) {
+        logger.info("Downloading the policy tosca model " + policyType + "/" + policyVersion);
         DumperOptions options = new DumperOptions();
         options.setDefaultScalarStyle(DumperOptions.ScalarStyle.PLAIN);
         options.setIndent(4);
@@ -171,11 +166,11 @@ public class PolicyEngineServices {
         Yaml yamlParser = new Yaml(options);
         String responseBody = callCamelRoute(
                 ExchangeBuilder.anExchange(camelContext).withProperty("policyModelName", policyType)
-                        .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-model",
+                        .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-tosca-model",
                 "Get one policy");
 
         if (responseBody == null || responseBody.isEmpty()) {
-            logger.warn("getPolicyModel returned by policy engine could not be decoded, as it's null or empty");
+            logger.warn("getPolicyToscaModel returned by policy engine could not be decoded, as it's null or empty");
             return null;
         }
 
diff --git a/src/main/java/org/onap/policy/clamp/policy/downloader/PolicyEngineController.java b/src/main/java/org/onap/policy/clamp/policy/downloader/PolicyEngineController.java
index 569678dc7..81775e5da 100644
--- a/src/main/java/org/onap/policy/clamp/policy/downloader/PolicyEngineController.java
+++ b/src/main/java/org/onap/policy/clamp/policy/downloader/PolicyEngineController.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP CLAMP
  * ================================================================================
- * Copyright (C) 2020 AT&T Intellectual Property. All rights
+ * Copyright (C) 2020-2021 AT&T Intellectual Property. All rights
  *                             reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,8 +27,8 @@ import com.att.eelf.configuration.EELFLogger;
 import com.att.eelf.configuration.EELFManager;
 import java.time.Instant;
 import org.json.simple.parser.ParseException;
-import org.onap.policy.clamp.clds.client.PolicyEngineServices;
 import org.onap.policy.clamp.loop.template.PolicyModelsRepository;
+import org.onap.policy.clamp.policy.PolicyEngineServices;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
diff --git a/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PdpGroupsAnalyzer.java b/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PdpGroupsAnalyzer.java
index 51cdb149e..83ece5795 100644
--- a/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PdpGroupsAnalyzer.java
+++ b/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PdpGroupsAnalyzer.java
@@ -1,6 +1,6 @@
 /*-
  * ============LICENSE_START=======================================================
- * ONAP CLAMP
+ * ONAP POLICY-CLAMP
  * ================================================================================
  * Copyright (C) 2021 AT&T Intellectual Property. All rights
  *                             reserved.
@@ -27,19 +27,19 @@ import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
+import java.util.Optional;
 import org.onap.policy.clamp.loop.template.PolicyModel;
 import org.onap.policy.models.pdp.concepts.PdpGroup;
 import org.onap.policy.models.pdp.concepts.PdpGroups;
 import org.onap.policy.models.pdp.concepts.PdpSubGroup;
 import org.onap.policy.models.pdp.enums.PdpState;
-import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyTypeIdentifier;
-import org.onap.sdc.toscaparser.api.elements.PolicyType;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaPolicyIdentifier;
 
 /**
  * This is an utility class to do searching in pdp groups.
  */
 public class PdpGroupsAnalyzer {
+
     /**
      * Get supported subGroups based on the defined policy type and version for s specific PDPgroup.
      * It returns null if the Group is not ACTIVE or if the policytype/version has not been found in the PDPSubgroups.
@@ -51,7 +51,8 @@ public class PdpGroupsAnalyzer {
      * @see org.onap.policy.models.pdp.concepts.PdpGroup
      * @see org.onap.policy.models.pdp.enums.PdpState
      */
-    private static JsonObject getSupportedSubgroups(PdpGroup pdpGroup, String policyType, String version) {
+    private static JsonObject getSupportedPdpSubgroupsForModelType(PdpGroup pdpGroup, String policyType,
+                                                                   String version) {
         if (!PdpState.ACTIVE.equals(pdpGroup.getPdpGroupState())) {
             return null;
         }
@@ -65,10 +66,27 @@ public class PdpGroupsAnalyzer {
                 supportedSubgroups.add(pdpSubGroup.getPdpType());
             }
         });
-        if (supportedSubgroups.size() == 0) {
-            return null;
-        }
-        return supportedPdpGroup;
+        return supportedSubgroups.size() == 0 ? null : supportedPdpGroup;
+    }
+
+    /**
+     * This method retrieves all supported pdpGroups and subgroups for a specific policy type/version.
+     *
+     * @param pdpGroups  The PdpGroups object containing all PEF pdp groups info
+     * @param policyType The policy type that must be used for searching
+     * @param version    THe policy type version that must be used for searching
+     * @return It returns a JsonObject containing each pdpGroup and subgroups associated
+     */
+    public static JsonObject getSupportedPdpGroupsForModelType(PdpGroups pdpGroups, String policyType, String version) {
+        JsonObject supportedPdpGroups = new JsonObject();
+        JsonArray pdpGroupsArray = new JsonArray();
+        supportedPdpGroups.add("supportedPdpGroups", pdpGroupsArray);
+
+        pdpGroups.getGroups().stream().map(pdpGroup -> PdpGroupsAnalyzer.getSupportedPdpSubgroupsForModelType(pdpGroup,
+                policyType, version)).filter(Objects::nonNull)
+                .forEach(jsonPdpGroup -> pdpGroupsArray.add(jsonPdpGroup));
+
+        return pdpGroupsArray.size() != 0 ? supportedPdpGroups : null;
     }
 
     /**
@@ -77,18 +95,39 @@ public class PdpGroupsAnalyzer {
      * @param policyModelsList The list of Policy Models where each PolicyModel will be updated
      * @param pdpGroups        The PdpGroups containing all PDP group definition
      */
-    public static void updatePdpGroup(List<PolicyModel> policyModelsList, PdpGroups pdpGroups) {
+    public static void updatePdpGroupOfPolicyModels(List<PolicyModel> policyModelsList, PdpGroups pdpGroups) {
         policyModelsList.parallelStream().forEach(policyModel -> {
-            JsonObject jsonResult = new JsonObject();
-            JsonArray supportedPdpGroups = new JsonArray();
-            jsonResult.add("supportedPdpGroups", supportedPdpGroups);
-            policyModel.setPolicyPdpGroup(jsonResult);
-            pdpGroups.getGroups().stream().map(pdpGroup -> PdpGroupsAnalyzer.getSupportedSubgroups(pdpGroup,
-                    policyModel.getPolicyModelType(), policyModel.getVersion())).filter(Objects::nonNull)
-                    .forEach(jsonPdpGroup -> supportedPdpGroups.add(jsonPdpGroup));
-            if (supportedPdpGroups.size() == 0) {
-                policyModel.setPolicyPdpGroup(null);
-            }
+            policyModel.setPolicyPdpGroup(getSupportedPdpGroupsForModelType(pdpGroups, policyModel.getPolicyModelType(),
+                    policyModel.getVersion()));
         });
     }
+
+    /**
+     * This method searches for the PdpGroup/subgroup where the policy given is currently deployed.
+     *
+     * @param pdpGroups The pdpGroups info from PEF
+     * @param policyName The policy Id
+     * @param version The policy version
+     * @return It returns a JsonObject containing the pdpGroup/subgroup info
+     */
+    public static JsonObject getPdpGroupDeploymentOfOnePolicy(PdpGroups pdpGroups, String policyName, String version) {
+        JsonObject pdpGroupInfo = new JsonObject();
+        JsonObject assignedPdpGroups = new JsonObject();
+        pdpGroupInfo.add("pdpGroupInfo", assignedPdpGroups);
+
+        ToscaPolicyIdentifier toscaPolicyIdentifier = new ToscaPolicyIdentifier(policyName, version);
+        pdpGroups.getGroups().stream().anyMatch(pdpGroup ->
+                pdpGroup.getPdpSubgroups().stream().anyMatch(
+                        pdpSubGroup -> {
+                            if (pdpSubGroup.getPolicies() != null && pdpSubGroup.getPolicies()
+                                    .contains(toscaPolicyIdentifier)) {
+                                assignedPdpGroups.addProperty("pdpGroup", pdpGroup.getName());
+                                assignedPdpGroups.addProperty("pdpSubGroup", pdpSubGroup.getPdpType());
+                                return true;
+                            }
+                            return false;
+                        })
+        );
+        return assignedPdpGroups.entrySet().isEmpty() ? null : pdpGroupInfo;
+    }
 }
\ No newline at end of file
diff --git a/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PoliciesPdpMerger.java b/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PoliciesPdpMerger.java
new file mode 100644
index 000000000..4b94355ee
--- /dev/null
+++ b/src/main/java/org/onap/policy/clamp/policy/pdpgroup/PoliciesPdpMerger.java
@@ -0,0 +1,101 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * ONAP POLICY-CLAMP
+ * ================================================================================
+ * Copyright (C) 2021 AT&T Intellectual Property. All rights
+ *                             reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END============================================
+ * ===================================================================
+ *
+ */
+
+package org.onap.policy.clamp.policy.pdpgroup;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import java.util.Map;
+import java.util.stream.StreamSupport;
+import org.onap.policy.clamp.clds.util.JsonUtils;
+import org.onap.policy.models.pdp.concepts.PdpGroups;
+
+/**
+ * This is an utility class that contains methods to work on the different results provided by the PEF.
+ * Mainly used to aggregate the results.
+ */
+public class PoliciesPdpMerger {
+
+    /**
+     * This method extract the content of a policy without knowing the key (policy Id).
+     * This JsonElement normally contains only the policy ID then the content,
+     * there is only one member in the Json element.
+     *
+     * @param policyJsonElement The policy as JsonElement
+     * @return It return the content as JsonObject
+     */
+    public static JsonObject getPolicyContentOutOfJsonElement(JsonElement policyJsonElement) {
+        return policyJsonElement.getAsJsonObject()
+                .get(((String) policyJsonElement.getAsJsonObject().keySet().toArray()[0])).getAsJsonObject();
+    }
+
+    /**
+     * This method merges 2 JsonElement together. If the jsonToMerge is null nothign is changed.
+     *
+     * @param json The initial json that will received the data
+     * @param jsonToMerge The json that will be added to the first json object
+     */
+    public static void mergeJsonElement(JsonObject json, JsonObject jsonToMerge) {
+        if (jsonToMerge != null) {
+            jsonToMerge.entrySet().stream().forEach(entry -> json.add(entry.getKey(), entry.getValue()));
+        }
+    }
+
+    /**
+     * This method merges the result of the policy listing and the associated Pdp Group info.
+     * It can be seen as an enrichment of the policy listing.
+     *
+     * @param jsonPoliciesList The Json containing the policies from the PEF
+     * @param pdpGroupsJson    The json containing the PDP groups info from the PEF
+     * @return It returns a String containing the policies list enriched with PdpGroup info
+     */
+    public static String mergePoliciesAndPdpGroupStates(String jsonPoliciesList, String pdpGroupsJson) {
+        PdpGroups pdpGroups = JsonUtils.GSON.fromJson(pdpGroupsJson, PdpGroups.class);
+        JsonObject policiesListJson =
+                JsonUtils.GSON.fromJson(jsonPoliciesList, JsonObject.class).get("topology_template")
+                        .getAsJsonObject();
+        StreamSupport.stream(policiesListJson.get("policies").getAsJsonArray().spliterator(), true)
+                .forEach(policyJson -> enrichOnePolicy(pdpGroups, getPolicyContentOutOfJsonElement(policyJson)));
+        return policiesListJson.toString();
+    }
+
+    /**
+     * Enrich one policy json node object with pdpGroup info.
+     *
+     * @param pdpGroups      The pdpGroups from PEF to search the policy
+     * @param policyJsonNode The policy json node that must be enriched
+     */
+    private static void enrichOnePolicy(PdpGroups pdpGroups, JsonObject policyJsonNode) {
+        JsonObject deploymentPdpJson = PdpGroupsAnalyzer
+                .getPdpGroupDeploymentOfOnePolicy(pdpGroups, policyJsonNode.get("name").getAsString(),
+                        policyJsonNode.get("version").getAsString());
+        mergeJsonElement(policyJsonNode, deploymentPdpJson);
+
+        JsonObject supportedPdpGroupsJson = PdpGroupsAnalyzer
+                .getSupportedPdpGroupsForModelType(pdpGroups, policyJsonNode.get("type").getAsString(),
+                        policyJsonNode.get("type_version").getAsString());
+        mergeJsonElement(policyJsonNode, supportedPdpGroupsJson);
+    }
+}
diff --git a/src/main/resources/clds/camel/routes/policy-flows.xml b/src/main/resources/clds/camel/routes/policy-flows.xml
index 4d1eba614..7048a5e03 100644
--- a/src/main/resources/clds/camel/routes/policy-flows.xml
+++ b/src/main/resources/clds/camel/routes/policy-flows.xml
@@ -170,11 +170,11 @@
             </doFinally>
         </doTry>
     </route>
-    <route id="get-policy-model">
-        <from uri="direct:get-policy-model"/>
+    <route id="get-policy-tosca-model">
+        <from uri="direct:get-policy-tosca-model"/>
         <doTry>
             <log loggingLevel="INFO"
-                 message="Getting the policy model: ${exchangeProperty[policyModelName]}/${exchangeProperty[policyModelVersion]}"/>
+                 message="Getting the policy tosca model: ${exchangeProperty[policyModelName]}/${exchangeProperty[policyModelVersion]}"/>
             <to
                     uri="bean:org.onap.policy.clamp.flow.log.FlowLogOperation?method=invokeLog('Policy', 'Getting the policy model')"/>
             <setHeader headerName="CamelHttpMethod">