Support topic properties files.

They will be loaded automatically at startup.
Remove POLICY-PAP-PDP topic as is moved to a feature.
Remove PDPD-CONFIGURATION topic as it is not used.

Change-Id: I55629f885d61ce1cc4e3f24bcae5279e65a96f22
Issue-ID: POLICY-1610
Signed-off-by: Jorge Hernandez <jorge.hernandez-herrero@att.com>

4 files changed, 158 insertions, 79 deletions

diff --git a/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java b/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
index b5c6fe94..88987eab 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
@@ -29,6 +29,7 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Properties;
 
@@ -54,12 +55,33 @@ public class FileSystemPersistence implements SystemPersistence {
     public static final String CONTROLLER_SUFFIX_IDENTIFIER = "-controller";
 
     /**
+     * File Backup Suffix.
+     */
+    public static final String FILE_BACKUP_SUFFIX = ".bak";
+
+    /**
      * Policy controller properties file suffix.
      */
     public static final String PROPERTIES_FILE_CONTROLLER_SUFFIX =
             CONTROLLER_SUFFIX_IDENTIFIER + PROPERTIES_FILE_EXTENSION;
 
     /**
+     * Topic configuration suffix.
+     */
+    public static final String TOPIC_SUFFIX_IDENTIFIER = "-topic";
+
+    /**
+     * Policy controller properties file suffix.
+     */
+    public static final String PROPERTIES_FILE_TOPIC_SUFFIX = TOPIC_SUFFIX_IDENTIFIER + PROPERTIES_FILE_EXTENSION;
+
+    /**
+     * Policy topic properties file suffix.
+     */
+    public static final String PROPERTIES_FILE_TOPIC_BACKUP_SUFFIX =
+        TOPIC_SUFFIX_IDENTIFIER + PROPERTIES_FILE_EXTENSION + ".bak";
+
+    /**
      * Policy controller properties file suffix.
      */
     public static final String PROPERTIES_FILE_CONTROLLER_BACKUP_SUFFIX =
@@ -228,6 +250,16 @@ public class FileSystemPersistence implements SystemPersistence {
         return getPropertiesList(PROPERTIES_FILE_CONTROLLER_SUFFIX, this::testControllerName);
     }
 
+    @Override
+    public Properties getTopicProperties(String topicName) {
+        return this.getProperties(topicName + TOPIC_SUFFIX_IDENTIFIER);
+    }
+
+    @Override
+    public List<Properties> getTopicProperties() {
+        return getPropertiesList(PROPERTIES_FILE_TOPIC_SUFFIX);
+    }
+
     private boolean testControllerName(String controllerFilename, Properties controllerProperties) {
         String controllerName = controllerFilename
                 .substring(0, controllerFilename.length() - PROPERTIES_FILE_CONTROLLER_SUFFIX.length());
@@ -245,79 +277,99 @@ public class FileSystemPersistence implements SystemPersistence {
 
     @Override
     public boolean backupController(String controllerName) {
-        final Path controllerPropertiesPath = Paths.get(this.configurationDirectory.toString(),
-                controllerName + PROPERTIES_FILE_CONTROLLER_SUFFIX);
+        return backup(controllerName, PROPERTIES_FILE_CONTROLLER_SUFFIX);
+    }
+
+    @Override
+    public boolean backupTopic(String topicName) {
+        return backup(topicName, PROPERTIES_FILE_TOPIC_SUFFIX);
+    }
 
-        if (Files.exists(controllerPropertiesPath)) {
+    protected boolean backup(String name, String fileSuffix) {
+        Path path = Paths.get(this.configurationDirectory.toString(), name + fileSuffix);
+        if (Files.exists(path)) {
             try {
-                logger.info("{}: there is an existing configuration file @ {} ", this,
-                        controllerPropertiesPath);
-                final Path controllerPropertiesBakPath = Paths.get(this.configurationDirectory.toString(),
-                        controllerName + PROPERTIES_FILE_CONTROLLER_BACKUP_SUFFIX);
-                Files.copy(controllerPropertiesPath, controllerPropertiesBakPath,
-                        StandardCopyOption.REPLACE_EXISTING);
-            } catch (final Exception e) {
-                logger.warn("{}: {} cannot be backed up", this, controllerName, e);
+                logger.info("{}: there is an existing configuration file @ {} ", this, path);
+                Path bakPath = Paths.get(this.configurationDirectory.toString(), 
+                                            name + fileSuffix + FILE_BACKUP_SUFFIX);
+                Files.copy(path, bakPath, StandardCopyOption.REPLACE_EXISTING);
+            } catch (Exception e) {
+                logger.warn("{}: {} cannot be backed up", this, name, e);
                 return false;
             }
         }
-
         return true;
     }
 
     @Override
     public boolean storeController(String controllerName, Object configuration) {
-        if (!(configuration instanceof Properties)) {
-            throw new IllegalArgumentException(
-                    "configuration must be of type properties to be handled by this manager");
-        }
+        checkPropertiesParam(configuration);
+        return store(controllerName, (Properties) configuration, PROPERTIES_FILE_CONTROLLER_SUFFIX);
+    }
 
-        final Properties properties = (Properties) configuration;
+    @Override
+    public boolean storeTopic(String topicName, Object configuration) {
+        checkPropertiesParam(configuration);
+        return store(topicName, (Properties) configuration, PROPERTIES_FILE_TOPIC_SUFFIX);
+    }
 
-        final Path controllerPropertiesPath = Paths.get(this.configurationDirectory.toString(),
-                controllerName + PROPERTIES_FILE_CONTROLLER_SUFFIX);
-        if (Files.exists(controllerPropertiesPath)) {
+    private boolean store(String name, Properties properties, String fileSuffix) {
+        Path path = Paths.get(this.configurationDirectory.toString(), name + fileSuffix);
+        if (Files.exists(path)) {
             try {
-                final Properties oldProperties =
-                        PropertyUtil.getProperties(controllerPropertiesPath.toFile());
+                Properties oldProperties = PropertyUtil.getProperties(path.toFile());
                 if (oldProperties.equals(properties)) {
-                    logger.info(
-                            "{}: noop: a properties file with the same contents exists for controller {}.", this,
-                            controllerName);
+                    logger.info("{}: noop: a properties file with the same contents exists for controller {}.", this,
+                                name);
                     return true;
                 } else {
-                    this.backupController(controllerName);
+                    this.backupController(name);
                 }
-            } catch (final Exception e) {
-                logger.info("{}: no existing {} properties {}", this, controllerName, e);
+            } catch (Exception e) {
+                logger.info("{}: no existing {} properties {}", this, name, e);
                 // continue
             }
         }
 
-        final File controllerPropertiesFile = controllerPropertiesPath.toFile();
-        try (FileWriter writer = new FileWriter(controllerPropertiesFile)) {
+        File file = path.toFile();
+        try (FileWriter writer = new FileWriter(file)) {
             properties.store(writer, "Machine created Policy Controller Configuration");
-        } catch (final Exception e) {
-            logger.warn("{}: {} cannot be saved", this, controllerName, e);
+        } catch (Exception e) {
+            logger.warn("{}: {} cannot be saved", this, name, e);
             return false;
         }
 
         return true;
     }
 
+    private void checkPropertiesParam(Object configuration) {
+        if (!(configuration instanceof Properties)) {
+            throw new IllegalArgumentException(
+                "configuration must be of type properties to be handled by this manager");
+        }
+    }
+
+
     @Override
     public boolean deleteController(String controllerName) {
-        final Path controllerPropertiesPath = Paths.get(this.configurationDirectory.toString(),
-                controllerName + PROPERTIES_FILE_CONTROLLER_SUFFIX);
+        return delete(controllerName, PROPERTIES_FILE_CONTROLLER_SUFFIX);
+    }
 
-        if (Files.exists(controllerPropertiesPath)) {
+    @Override
+    public boolean deleteTopic(String topicName) {
+        return delete(topicName, PROPERTIES_FILE_TOPIC_SUFFIX);
+    }
+
+    protected boolean delete(String name, String fileSuffix) {
+        Path path = Paths.get(this.configurationDirectory.toString(), name + fileSuffix);
+
+        if (Files.exists(path)) {
             try {
-                final Path controllerPropertiesBakPath = Paths.get(this.configurationDirectory.toString(),
-                        controllerName + PROPERTIES_FILE_CONTROLLER_BACKUP_SUFFIX);
-                Files.move(controllerPropertiesPath, controllerPropertiesBakPath,
-                        StandardCopyOption.REPLACE_EXISTING);
+                Path bakPath = Paths.get(this.configurationDirectory.toString(), 
+                                            name + fileSuffix + FILE_BACKUP_SUFFIX);
+                Files.move(path, bakPath, StandardCopyOption.REPLACE_EXISTING);
             } catch (final Exception e) {
-                logger.warn("{}: {} cannot be deleted", this, controllerName, e);
+                logger.warn("{}: {} cannot be deleted", this, name, e);
                 return false;
             }
         }
@@ -328,17 +380,18 @@ public class FileSystemPersistence implements SystemPersistence {
     /**
      * provides a list of files sorted by name in ascending order in the configuration directory.
      */
-    protected File[] sortedListFiles() {
-        final File[] dirFiles = this.configurationDirectory.toFile().listFiles();
-        Arrays.sort(dirFiles, (e1, e2) -> e1.getName().compareTo(e2.getName()));
+    private File[] sortedListFiles() {
+        File[] dirFiles = this.configurationDirectory.toFile().listFiles();
+        if (dirFiles != null) {
+            Arrays.sort(dirFiles, Comparator.comparing(File::getName));
+        } else {
+            dirFiles = new File[]{};
+        }
         return dirFiles;
     }
 
     @Override
     public String toString() {
-        final StringBuilder builder = new StringBuilder();
-        builder.append("FileSystemPersistence [configurationDirectory=")
-        .append(this.configurationDirectory).append("]");
-        return builder.toString();
+        return "FileSystemPersistence [configurationDirectory=" + this.configurationDirectory + "]";
     }
 }
diff --git a/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java b/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
index 32852225..ee1e8b52 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
@@ -67,6 +67,14 @@ public interface SystemPersistence {
     boolean backupController(String controllerName);
 
     /**
+     * backs up a topic configuration.
+     *
+     * @param topicName the controller name
+     * @return true if the configuration is backed up
+     */
+    boolean backupTopic(String topicName);
+
+    /**
      * persists controller configuration.
      *
      * @param controllerName the controller name
@@ -79,6 +87,18 @@ public interface SystemPersistence {
     boolean storeController(String controllerName, Object configuration);
 
     /**
+     * persists topic configuration.
+     *
+     * @param topicName the controller name
+     * @param configuration object containing the configuration
+     *
+     * @return true if storage is succesful, false otherwise
+     * @throws IllegalArgumentException if the configuration cannot be handled by the persistence
+     *         manager
+     */
+    boolean storeTopic(String topicName, Object configuration);
+
+    /**
      * delete controller configuration.
      *
      * @param controllerName the controller name
@@ -87,6 +107,21 @@ public interface SystemPersistence {
     boolean deleteController(String controllerName);
 
     /**
+     * delete topic configuration.
+     *
+     * @param topicName the topic name
+     * @return true if storage is succesful, false otherwise
+     */
+    boolean deleteTopic(String topicName);
+
+    /**
+     * get controllers configuration.
+     *
+     * @return list of controllers properties
+     */
+    List<Properties> getControllerProperties();
+
+    /**
      * get controller properties.
      *
      * @param controllerName controller name
@@ -98,11 +133,21 @@ public interface SystemPersistence {
     Properties getControllerProperties(String controllerName);
 
     /**
-     * get controllers configuration.
+     * get topic configuration.
      *
-     * @return list of controllers properties
+     * @return list of topic properties
      */
-    List<Properties> getControllerProperties();
+    List<Properties> getTopicProperties();
+
+    /**
+     * get topic properties.
+     *
+     * @param topicName topic name
+     * @return properties for this topic
+     *
+     * @throws IllegalArgumentException if topicName is invalid
+     */
+    Properties getTopicProperties(String topicName);
 
     /**
      * get environments.
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/Main.java b/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
index 5ebd0490..38af4138 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
@@ -21,6 +21,7 @@
 package org.onap.policy.drools.system;
 
 import java.util.Properties;
+import org.onap.policy.common.endpoints.event.comm.TopicEndpoint;
 import org.onap.policy.drools.persistence.SystemPersistence;
 import org.onap.policy.drools.properties.DroolsProperties;
 import org.onap.policy.drools.utils.PropertyUtil;
@@ -76,7 +77,13 @@ public class Main {
             PolicyEngine.manager.setEnvironment(env);
         }
 
-        /* 2. Start the Engine with the basic services only (no Policy Controllers) */
+        /* 2. Add topics */
+
+        for (Properties topicProperties : SystemPersistence.manager.getTopicProperties()) {
+            TopicEndpoint.manager.addTopics(topicProperties);
+        }
+
+        /* 3. Start the Engine with the basic services only (no Policy Controllers) */
 
         MDCTransaction trans =
                 MDCTransaction.newTransaction(null, null)
@@ -121,7 +128,7 @@ public class Main {
             System.exit(1);
         }
 
-        /* 3. Create and start the controllers */
+        /* 4. Create and start the controllers */
 
         for (final Properties controllerProperties :
             SystemPersistence.manager.getControllerProperties()) {
diff --git a/policy-management/src/main/server/config/engine.properties b/policy-management/src/main/server/config/engine.properties
index fec99b14..05f682e3 100644
--- a/policy-management/src/main/server/config/engine.properties
+++ b/policy-management/src/main/server/config/engine.properties
@@ -20,9 +20,7 @@
 
 # Policy Engine Configuration Channels
 
-dmaap.source.topics=PDPD-CONFIGURATION,POLICY-PDP-PAP
-
-dmaap.sink.topics=PDPD-CONFIGURATION,POLICY-PDP-PAP
+dmaap.source.topics=PDPD-CONFIGURATION
 
 dmaap.source.topics.PDPD-CONFIGURATION.servers=${env:DMAAP_SERVERS}
 dmaap.source.topics.PDPD-CONFIGURATION.effectiveTopic=${env:PDPD_CONFIGURATION_TOPIC}
@@ -33,30 +31,6 @@ dmaap.source.topics.PDPD-CONFIGURATION.consumerInstance=${env:PDPD_CONFIGURATION
 dmaap.source.topics.PDPD-CONFIGURATION.managed=false
 dmaap.source.topics.PDPD-CONFIGURATION.https=true
 
-dmaap.sink.topics.PDPD-CONFIGURATION.servers=${env:DMAAP_SERVERS}
-dmaap.sink.topics.PDPD-CONFIGURATION.effectiveTopic=${env:PDPD_CONFIGURATION_TOPIC}
-dmaap.sink.topics.PDPD-CONFIGURATION.apiKey=${env:PDPD_CONFIGURATION_API_KEY}
-dmaap.sink.topics.PDPD-CONFIGURATION.apiSecret=${env:PDPD_CONFIGURATION_API_SECRET}
-dmaap.sink.topics.PDPD-CONFIGURATION.partitionKey=${env:PDPD_CONFIGURATION_PARTITION_KEY}
-dmaap.sink.topics.PDPD-CONFIGURATION.managed=false
-dmaap.sink.topics.PDPD-CONFIGURATION.https=true
-
-# Configuration Channel Settings: POLICY-PDP-PAP
-
-dmaap.source.topics.POLICY-PDP-PAP.servers=${env:DMAAP_SERVERS}
-dmaap.source.topics.POLICY-PDP-PAP.effectiveTopic=${env:POLICY_PDP_PAP_TOPIC}
-dmaap.source.topics.POLICY-PDP-PAP.apiKey=${env:POLICY_PDP_PAP_API_KEY}
-dmaap.source.topics.POLICY-PDP-PAP.apiSecret=${env:POLICY_PDP_PAP_API_SECRET}
-dmaap.source.topics.POLICY-PDP-PAP.managed=false
-dmaap.source.topics.POLICY-PDP-PAP.https=true
-
-dmaap.sink.topics.POLICY-PDP-PAP.servers=${env:DMAAP_SERVERS}
-dmaap.sink.topics.POLICY-PDP-PAP.effectiveTopic=${env:POLICY_PDP_PAP_TOPIC}
-dmaap.sink.topics.POLICY-PDP-PAP.apiKey=${env:POLICY_PDP_PAP_API_KEY}
-dmaap.sink.topics.POLICY-PDP-PAP.apiSecret=${env:POLICY_PDP_PAP_API_SECRET}
-dmaap.sink.topics.POLICY-PDP-PAP.managed=false
-dmaap.sink.topics.POLICY-PDP-PAP.https=true
-
 http.server.services=SECURED-CONFIG
 
 http.server.services.SECURED-CONFIG.host=${env:TELEMETRY_HOST}