Add policy downloader

Add policy downloader to sync all policy types in the database

Issue-ID: CLAMP-518
Change-Id: I5ab82970cd3403e46fe7cc8447766977b11b68e7
Signed-off-by: sebdet <sebastien.determe@intl.att.com>

9 files changed, 160 insertions, 63 deletions

diff --git a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
index d99e9b56..96294207 100644
--- a/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
+++ b/src/main/java/org/onap/clamp/clds/client/PolicyEngineServices.java
@@ -30,6 +30,8 @@ import org.apache.camel.CamelContext;
 import org.apache.camel.Exchange;
 import org.apache.camel.builder.ExchangeBuilder;
 import org.onap.clamp.clds.config.ClampProperties;
+import org.onap.clamp.clds.sdc.controller.installer.BlueprintMicroService;
+import org.onap.clamp.loop.template.PolicyModel;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -43,32 +45,52 @@ import org.springframework.stereotype.Component;
 public class PolicyEngineServices {
     private final CamelContext camelContext;
 
-    private final ClampProperties refProp;
+    private static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyEngineServices.class);
+    private static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
+    private static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
+    private static int retryInterval = 0;
+    private static int retryLimit = 1;
 
-    protected static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyEngineServices.class);
-    protected static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
-    protected static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
     public static final String POLICY_RETRY_INTERVAL = "policy.retry.interval";
     public static final String POLICY_RETRY_LIMIT = "policy.retry.limit";
 
     @Autowired
     public PolicyEngineServices(CamelContext camelContext, ClampProperties refProp) {
-        this.refProp = refProp;
         this.camelContext = camelContext;
+
+        if (refProp.getStringValue(POLICY_RETRY_LIMIT) != null) {
+            retryLimit = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_LIMIT));
+        }
+        if (refProp.getStringValue(POLICY_RETRY_INTERVAL) != null) {
+            retryInterval = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_INTERVAL));
+        }
     }
 
-    private void downloadAllPolicies() {
-        /*
-         * Exchange myCamelExchange = ExchangeBuilder.anExchange(camelContext)
-         * .withProperty("blueprintResourceId",
-         * resourceUuid).withProperty("blueprintServiceId", serviceUuid)
-         * .withProperty("blueprintName", artifactName).build();
-         * metricsLogger.info("Attempt n°" + i + " to contact DCAE inventory");
-         * 
-         * Exchange exchangeResponse =
-         * camelContext.createProducerTemplate().send("direct:get-all-policy-models",
-         * myCamelExchange);
-         */
+    public PolicyModel createPolicyModelFromPolicyEngine(String policyType, String policyVersion)
+            throws InterruptedException {
+        return new PolicyModel(policyType, this.downloadOnePolicy(policyType, policyVersion), policyVersion,
+                createPolicyAcronym(policyType));
+    }
+
+    public PolicyModel createPolicyModelFromPolicyEngine(BlueprintMicroService microService)
+            throws InterruptedException {
+        return createPolicyModelFromPolicyEngine(microService.getModelType(), microService.getModelVersion());
+    }
+
+    private static String createPolicyAcronym(String policyType) {
+        String[] policyNameArray = policyType.split("\\.");
+        return policyNameArray[policyNameArray.length - 1];
+    }
+
+    /**
+     * This method can be used to download all policy types + data types defined in
+     * policy engine.
+     * 
+     * @return A yaml containing all policy Types and all data types
+     * @throws InterruptedException In case of issue when sleeping during the retry
+     */
+    public String downloadAllPolicies() throws InterruptedException {
+        return callCamelRoute(ExchangeBuilder.anExchange(camelContext).build(), "direct:get-all-policy-models");
     }
 
     /**
@@ -77,34 +99,24 @@ public class PolicyEngineServices {
      * @param policyType    The policy type (id)
      * @param policyVersion The policy version
      * @return A string with the whole policy tosca model
-     * @throws InterruptedException in case of issue when sleeping during the retry
+     * @throws InterruptedException In case of issue when sleeping during the retry
      */
     public String downloadOnePolicy(String policyType, String policyVersion) throws InterruptedException {
-        int retryInterval = 0;
-        int retryLimit = 1;
-        if (refProp.getStringValue(POLICY_RETRY_LIMIT) != null) {
-            retryLimit = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_LIMIT));
-        }
-        if (refProp.getStringValue(POLICY_RETRY_INTERVAL) != null) {
-            retryInterval = Integer.valueOf(refProp.getStringValue(POLICY_RETRY_INTERVAL));
-        }
-        for (int i = 0; i < retryLimit; i++) {
-            Exchange paramExchange = ExchangeBuilder.anExchange(camelContext)
-                    .withProperty("policyModelName", policyType).withProperty("policyModelVersion", policyVersion)
-                    .build();
-
-            Exchange exchangeResponse = camelContext.createProducerTemplate().send("direct:get-policy-model",
-                    paramExchange);
+        return callCamelRoute(ExchangeBuilder.anExchange(camelContext).withProperty("policyModelName", policyType)
+                .withProperty("policyModelVersion", policyVersion).build(), "direct:get-policy-model");
+    }
 
+    private String callCamelRoute(Exchange exchange, String camelFlow) throws InterruptedException {
+        for (int i = 0; i < retryLimit; i++) {
+            Exchange exchangeResponse = camelContext.createProducerTemplate().send(camelFlow, exchange);
             if (Integer.valueOf(200).equals(exchangeResponse.getIn().getHeader("CamelHttpResponseCode"))) {
                 return (String) exchangeResponse.getIn().getBody();
             } else {
-                logger.info("Policy " + retryInterval + "ms before retrying ...");
+                logger.info("Policy query " + retryInterval + "ms before retrying ...");
                 // wait for a while and try to connect to DCAE again
                 Thread.sleep(retryInterval);
             }
         }
         return "";
     }
-
 }
diff --git a/src/main/java/org/onap/clamp/loop/CsarInstaller.java b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
index 022b0e28..c0cfac96 100644
--- a/src/main/java/org/onap/clamp/loop/CsarInstaller.java
+++ b/src/main/java/org/onap/clamp/loop/CsarInstaller.java
@@ -183,21 +183,10 @@ public class CsarInstaller {
         return newSet;
     }
 
-    private static String createPolicyAcronym(String policyType) {
-        String[] policyNameArray = policyType.split("\\.");
-        return policyNameArray[policyNameArray.length - 1];
-    }
-
-    private PolicyModel createPolicyModel(BlueprintMicroService microService) throws InterruptedException {
-        return new PolicyModel(microService.getModelType(),
-                policyEngineServices.downloadOnePolicy(microService.getModelType(), microService.getModelVersion()),
-                microService.getModelVersion(), createPolicyAcronym(microService.getModelType()));
-    }
-
     private PolicyModel getPolicyModel(BlueprintMicroService microService) throws InterruptedException {
         return policyModelsRepository
                 .findById(new PolicyModelId(microService.getModelType(), microService.getModelVersion()))
-                .orElse(createPolicyModel(microService));
+                .orElse(policyEngineServices.createPolicyModelFromPolicyEngine(microService));
     }
 
     /**
diff --git a/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java b/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
index b712dc3f..8795a125 100644
--- a/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
+++ b/src/main/java/org/onap/clamp/policy/downloader/PolicyDownloader.java
@@ -26,12 +26,18 @@ package org.onap.clamp.policy.downloader;
 import com.att.eelf.configuration.EELFLogger;
 import com.att.eelf.configuration.EELFManager;
 
-import org.apache.camel.CamelContext;
-import org.onap.clamp.clds.client.DcaeInventoryServices;
-import org.onap.clamp.clds.config.ClampProperties;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+
+import org.onap.clamp.clds.client.PolicyEngineServices;
+import org.onap.clamp.loop.template.PolicyModel;
+import org.onap.clamp.loop.template.PolicyModelId;
+import org.onap.clamp.loop.template.PolicyModelsRepository;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.yaml.snakeyaml.Yaml;
 
 /**
  * This class implements a periodic job that is done in the background to
@@ -42,20 +48,49 @@ import org.springframework.context.annotation.Profile;
 @Profile("clamp-policy-controller")
 public class PolicyDownloader {
 
-    protected static final EELFLogger logger = EELFManager.getInstance().getLogger(DcaeInventoryServices.class);
+    protected static final EELFLogger logger = EELFManager.getInstance().getLogger(PolicyDownloader.class);
     protected static final EELFLogger auditLogger = EELFManager.getInstance().getAuditLogger();
     protected static final EELFLogger metricsLogger = EELFManager.getInstance().getMetricsLogger();
     public static final String POLICY_RETRY_INTERVAL = "policy.retry.interval";
     public static final String POLICY_RETRY_LIMIT = "policy.retry.limit";
 
-    private final CamelContext camelContext;
-
-    private final ClampProperties refProp;
+    private final PolicyEngineServices policyEngineServices;
+    private final PolicyModelsRepository policyModelsRepository;
 
     @Autowired
-    public PolicyDownloader(CamelContext camelContext, ClampProperties refProp) {
-        this.refProp = refProp;
-        this.camelContext = camelContext;
+    public PolicyDownloader(PolicyEngineServices policyEngineService, PolicyModelsRepository policyModelsRepository) {
+        this.policyEngineServices = policyEngineService;
+        this.policyModelsRepository = policyModelsRepository;
+    }
+
+    private void createPolicyInDbIfNeeded(PolicyModel policyModel) {
+        if (!policyModelsRepository
+                .existsById(new PolicyModelId(policyModel.getPolicyModelType(), policyModel.getVersion()))) {
+            policyModelsRepository.save(policyModel);
+        }
+    }
+
+    @Scheduled(fixedRate = 120000)
+    public void synchronizeAllPolicies() throws InterruptedException {
+        try {
+            LinkedHashMap<String, Object> loadedYaml = new Yaml().load(policyEngineServices.downloadAllPolicies());
+            if (loadedYaml == null || loadedYaml.isEmpty()) {
+                logger.warn(
+                        "getAllPolicyType yaml returned by policy engine could not be decoded, as it's null or empty");
+                return;
+            }
+
+            LinkedHashMap<String, Object> policyTypesList = (LinkedHashMap<String, Object>) loadedYaml
+                    .get("policy_types");
+            for (Entry<String, Object> policyType : policyTypesList.entrySet()) {
+                createPolicyInDbIfNeeded(policyEngineServices.createPolicyModelFromPolicyEngine(policyType.getKey(),
+                        ((String) ((LinkedHashMap<String, Object>) policyType.getValue()).get("version"))));
+            }
+        } catch (InterruptedException e) {
+            logger.warn("query to policy engine has been interrupted", e);
+            throw e;
+        }
+
     }
 
 }
diff --git a/src/main/resources/application-noaaf.properties b/src/main/resources/application-noaaf.properties
index b9af1b47..69d16875 100644
--- a/src/main/resources/application-noaaf.properties
+++ b/src/main/resources/application-noaaf.properties
@@ -73,7 +73,7 @@ clamp.config.keyFile=classpath:/clds/aaf/org.onap.clamp.keyfile
 
 server.servlet.context-path=/
 #Modified engine-rest applicationpath
-spring.profiles.active=clamp-default,clamp-default-user,clamp-sdc-controller,clamp-ssl-config
+spring.profiles.active=clamp-default,clamp-default-user,clamp-sdc-controller,clamp-ssl-config,clamp-policy-controller
 spring.http.converters.preferred-json-mapper=gson
 
 #The max number of active threads in this pool
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
index e4568995..bb25abff 100644
--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -78,7 +78,7 @@ server.ssl.trust-store-password=enc:iDnPBBLq_EMidXlMa1FEuBR8TZzYxrCg66vq_XfLHdJ
 
 server.servlet.context-path=/
 #Modified engine-rest applicationpath
-spring.profiles.active=clamp-default,clamp-aaf-authentication,clamp-sdc-controller,clamp-ssl-config
+spring.profiles.active=clamp-default,clamp-aaf-authentication,clamp-sdc-controller,clamp-ssl-config,clamp-policy-controller
 spring.http.converters.preferred-json-mapper=gson
 
 #The max number of active threads in this pool
diff --git a/src/main/resources/clds/camel/routes/policy-flows.xml b/src/main/resources/clds/camel/routes/policy-flows.xml
index ce24b27c..c28e4543 100644
--- a/src/main/resources/clds/camel/routes/policy-flows.xml
+++ b/src/main/resources/clds/camel/routes/policy-flows.xml
@@ -140,7 +140,7 @@
 						<log loggingLevel="INFO"
 								message="Endpoint to get all policy models: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes"></log>
 						<toD
-								uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
+								uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
 						<convertBodyTo type="java.lang.String"/>		
 						<doFinally>
 								<to uri="direct:reset-raise-http-exception-flag"/>
@@ -174,7 +174,7 @@
 						<log loggingLevel="INFO"
 								message="Endpoint to get policy model: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}"></log>
 						<toD
-								uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
+								uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
 					    <convertBodyTo type="java.lang.String"/>
 						<doFinally>
 								<to uri="direct:reset-raise-http-exception-flag"/>
diff --git a/src/test/resources/clds/camel/routes/policy-flows.xml b/src/test/resources/clds/camel/routes/policy-flows.xml
index ce24b27c..c28e4543 100644
--- a/src/test/resources/clds/camel/routes/policy-flows.xml
+++ b/src/test/resources/clds/camel/routes/policy-flows.xml
@@ -140,7 +140,7 @@
 						<log loggingLevel="INFO"
 								message="Endpoint to get all policy models: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes"></log>
 						<toD
-								uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
+								uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
 						<convertBodyTo type="java.lang.String"/>		
 						<doFinally>
 								<to uri="direct:reset-raise-http-exception-flag"/>
@@ -174,7 +174,7 @@
 						<log loggingLevel="INFO"
 								message="Endpoint to get policy model: {{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}"></log>
 						<toD
-								uri="{{clamp.config.policy.pap.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
+								uri="{{clamp.config.policy.api.url}}/policy/api/v1/policytypes/${exchangeProperty[policyModelName]}/versions/${exchangeProperty[policyModelVersion]}?bridgeEndpoint=true&amp;useSystemProperties=true&amp;throwExceptionOnFailure=${exchangeProperty[raiseHttpExceptionFlag]}&amp;authMethod=Basic&amp;authUsername={{clamp.config.policy.api.userName}}&amp;authPassword={{clamp.config.policy.api.password}}&amp;connectionTimeToLive=5000&amp;httpClient.connectTimeout=10000&amp;httpClient.socketTimeout=20000&amp;authenticationPreemptive=true&amp;connectionClose=true"/>
 					    <convertBodyTo type="java.lang.String"/>
 						<doFinally>
 								<to uri="direct:reset-raise-http-exception-flag"/>
diff --git a/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file
new file mode 100644
index 00000000..7394d3f9
--- /dev/null
+++ b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.file
@@ -0,0 +1,60 @@
+tosca_definitions_version: tosca_simple_yaml_1_0_0
+policy_types:
+  - onap.policies.Monitoring:
+      version: 1.0.0
+      description: A base policy type for all policies that govern monitoring provision
+      derived_from: tosca.policies.Root
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+ - onap.policies.controlloop.Operational:
+      version: 1.0.0
+      description: Operational Policy for Control Loops
+      derived_from: tosca.policies.Root
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+  - onap.policies.controloop.operational.Drools:
+      version: 1.0.0
+      description: Operational Policy for Control Loops using the Drools PDP
+      derived_from: onap.policies.controlloop.Operational
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+  - onap.policies.controloop.operational.Apex:
+      version: 1.0.0
+      description: Operational Policy for Control Loops using the APEX PDP
+      derived_from: onap.policies.controlloop.Operational
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+ - onap.policies.controlloop.Guard:
+      version: 1.0.0
+      description: Operational Policy for Control Loops
+      derived_from: tosca.policies.Root
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+  - onap.policies.controlloop.guard.FrequencyLimiter:
+      version: 1.0.0
+      description: Supports limiting the frequency of actions being taken by a Actor.
+      derived_from: onap.policies.controlloop.Guard
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+  - onap.policies.controlloop.guard.Blacklist:
+      version: 1.0.0
+      description: Supports blacklist of VNF's from performing control loop actions on.
+      derived_from: onap.policies.controlloop.Guard
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+  - onap.policies.controlloop.guard.MinMax:
+      version: 1.0.0
+      description: Supports Min/Max number of VF Modules
+      derived_from: onap.policies.controlloop.Guard
+      properties:
+        # Omitted for brevity, see Section 1
+ 
+data_types:
+  # Any bespoke data types referenced by policy type definitions[]
diff --git a/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header
new file mode 100644
index 00000000..6a280d97
--- /dev/null
+++ b/src/test/resources/http-cache/example/policy/api/v1/policytypes/.header
@@ -0,0 +1 @@
+{"Transfer-Encoding": "chunked", "Set-Cookie": "JSESSIONID=158qxkdtdobkd1umr3ikkgrmlx;Path=/", "Expires": "Thu, 01 Jan 1970 00:00:00 GMT", "Server": "Jetty(9.3.21.v20170918)", "Content-Type": "application/json", "X-ECOMP-RequestID": "e2ddb3c8-994f-47df-b4dc-097d4fb55c08"}
\ No newline at end of file