move all hard install config to environment vars

+ support multiple system properties files with variable interpolation
  loaded at initialization
+ support of configurable JVM options (-X, etc ..).
+ rearrange aaf configuration to avoid {{}} installation variables
  and use dynamic enviroment variables.
+ miscellaneous clean up in areas touched and checkstyle.

Change-Id: I71ad839778e17eb57c098a2c5cc2bf96e468669a
Issue-ID: POLICY-1524
Signed-off-by: Jorge Hernandez <jorge.hernandez-herrero@att.com>

4 files changed, 193 insertions, 170 deletions

diff --git a/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java b/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
index 2fcb07d5..b5c6fe94 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/persistence/FileSystemPersistence.java
@@ -1,8 +1,8 @@
 /*
  * ============LICENSE_START=======================================================
- * policy-management
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 
+import java.util.function.BiPredicate;
 import org.onap.policy.drools.properties.DroolsProperties;
 import org.onap.policy.drools.utils.PropertyUtil;
 import org.slf4j.Logger;
@@ -41,45 +42,60 @@ import org.slf4j.LoggerFactory;
  * Properties based Persistence.
  */
 public class FileSystemPersistence implements SystemPersistence {
-    
+
+    /**
+     * Properties file extension.
+     */
+    public static final String PROPERTIES_FILE_EXTENSION = ".properties";
+
     /**
-     * policy controllers suffix.
+     * Policy controllers suffix.
      */
     public static final String CONTROLLER_SUFFIX_IDENTIFIER = "-controller";
 
     /**
-     * policy controller properties file suffix.
+     * Policy controller properties file suffix.
      */
     public static final String PROPERTIES_FILE_CONTROLLER_SUFFIX =
-            CONTROLLER_SUFFIX_IDENTIFIER + ".properties";
+            CONTROLLER_SUFFIX_IDENTIFIER + PROPERTIES_FILE_EXTENSION;
 
     /**
-     * policy controller properties file suffix.
+     * Policy controller properties file suffix.
      */
     public static final String PROPERTIES_FILE_CONTROLLER_BACKUP_SUFFIX =
-            CONTROLLER_SUFFIX_IDENTIFIER + ".properties.bak";
+            CONTROLLER_SUFFIX_IDENTIFIER + PROPERTIES_FILE_EXTENSION + ".bak";
 
     /**
-     * policy engine properties file name.
+     * Policy engine properties file name.
      */
-    public static final String PROPERTIES_FILE_ENGINE = "policy-engine.properties";
+    public static final String PROPERTIES_FILE_ENGINE = "engine" + PROPERTIES_FILE_EXTENSION;
 
     /**
      * Installation environment suffix for files.
      */
-    public static final String ENV_SUFFIX = ".environment";
+    public static final String ENV_FILE_SUFFIX = ".environment";
 
     /**
-     * configuration directory.
+     * Environment properties extension.
+     */
+    public static final String ENV_FILE_EXTENSION = ENV_FILE_SUFFIX;
+
+    /**
+     * Installation environment suffix for files.
+     */
+    public static final String SYSTEM_PROPERTIES_SUFFIX = "-system";
+    public static final String SYSTEM_PROPERTIES_FILE_SUFFIX = SYSTEM_PROPERTIES_SUFFIX + PROPERTIES_FILE_EXTENSION;
+
+    /**
+     * Configuration directory.
      */
     protected Path configurationDirectory = Paths.get(DEFAULT_CONFIGURATION_DIR);
 
     /**
-     * logger.
+     * Logger.
      */
     private static final Logger logger = LoggerFactory.getLogger(FileSystemPersistence.class);
 
-
     @Override
     public void setConfigurationDir(String configDir) {
         String tempConfigDir = configDir;
@@ -93,6 +109,11 @@ public class FileSystemPersistence implements SystemPersistence {
             this.configurationDirectory = Paths.get(tempConfigDir);
         }
 
+        setConfigurationDir();
+    }
+
+    @Override
+    public void setConfigurationDir() {
         if (Files.notExists(this.configurationDirectory)) {
             try {
                 Files.createDirectories(this.configurationDirectory);
@@ -103,7 +124,7 @@ public class FileSystemPersistence implements SystemPersistence {
 
         if (!Files.isDirectory(this.configurationDirectory)) {
             throw new IllegalStateException(
-                    "config directory: " + this.configurationDirectory + " is not a directory");
+                "config directory: " + this.configurationDirectory + " is not a directory");
         }
     }
 
@@ -112,22 +133,117 @@ public class FileSystemPersistence implements SystemPersistence {
         return this.configurationDirectory;
     }
 
+    protected Properties getProperties(Path propertiesPath) {
+        if (!Files.exists(propertiesPath)) {
+            throw new IllegalArgumentException("properties for " + propertiesPath.toString() + " are not persisted.");
+        }
+
+        try {
+            return PropertyUtil.getProperties(propertiesPath.toFile());
+        } catch (final Exception e) {
+            throw new IllegalArgumentException("can't read properties for " + propertiesPath.toString(), e);
+        }
+    }
+
     @Override
-    public Properties getEngineProperties() {
-        final Path policyEnginePath =
-                Paths.get(this.configurationDirectory.toString(), PROPERTIES_FILE_ENGINE);
+    public Properties getProperties(String name) {
+        if (name == null || name.isEmpty()) {
+            throw new IllegalArgumentException("properties name must be provided");
+        }
+
+        Path propertiesPath = Paths.get(this.configurationDirectory.toString());
+        if (name.endsWith(PROPERTIES_FILE_EXTENSION) || name.endsWith(ENV_FILE_EXTENSION)) {
+            propertiesPath = propertiesPath.resolve(name);
+        } else {
+            propertiesPath = propertiesPath.resolve(name + PROPERTIES_FILE_EXTENSION);
+        }
+
+        return getProperties(propertiesPath);
+    }
+
+    protected List<Properties> getPropertiesList(String suffix) {
+        return getPropertiesList(suffix, (name, props) ->  true);
+    }
+
+    protected List<Properties> getPropertiesList(String suffix, BiPredicate<String, Properties> preCondition) {
+        List<Properties> properties = new ArrayList<>();
+        File[] files = this.sortedListFiles();
+        for (File file : files) {
+            if (file.getName().endsWith(suffix)) {
+                addToPropertiesList(file, properties, preCondition);
+            }
+        }
+        return properties;
+    }
+
+    private void addToPropertiesList(File file, List<Properties> properties,
+                                     BiPredicate<String, Properties> preCondition) {
         try {
-            if (Files.exists(policyEnginePath)) {
-                return PropertyUtil.getProperties(policyEnginePath.toFile());
+            Properties proposedProps = getProperties(file.getName());
+            if (preCondition.test(file.getName(), proposedProps)) {
+                properties.add(proposedProps);
             }
         } catch (final Exception e) {
-            logger.warn("{}: could not find {}", this, policyEnginePath, e);
+            logger.error("{}: cannot get properties {} because of {}", this, file.getName(),
+                e.getMessage(), e);
         }
+    }
 
-        return null;
+    @Override
+    public Properties getEnvironmentProperties(String name) {
+        if (name == null || name.isEmpty()) {
+            throw new IllegalArgumentException("environment name must be provided");
+        }
+
+        return this.getProperties(Paths.get(this.configurationDirectory.toString(), name + ENV_FILE_SUFFIX));
     }
 
     @Override
+    public List<Properties> getEnvironmentProperties() {
+        return getPropertiesList(ENV_FILE_SUFFIX);
+    }
+
+    @Override
+    public Properties getSystemProperties(String name) {
+        return this.getProperties(name + SYSTEM_PROPERTIES_SUFFIX);
+    }
+
+    @Override
+    public List<Properties> getSystemProperties() {
+        return getPropertiesList(SYSTEM_PROPERTIES_FILE_SUFFIX);
+    }
+
+    @Override
+    public Properties getEngineProperties() {
+        return this.getProperties(PROPERTIES_FILE_ENGINE);
+    }
+
+    @Override
+    public Properties getControllerProperties(String controllerName) {
+        return this.getProperties(controllerName + CONTROLLER_SUFFIX_IDENTIFIER);
+    }
+
+    @Override
+    public List<Properties> getControllerProperties() {
+        return getPropertiesList(PROPERTIES_FILE_CONTROLLER_SUFFIX, this::testControllerName);
+    }
+
+    private boolean testControllerName(String controllerFilename, Properties controllerProperties) {
+        String controllerName = controllerFilename
+                .substring(0, controllerFilename.length() - PROPERTIES_FILE_CONTROLLER_SUFFIX.length());
+        String controllerPropName = controllerProperties.getProperty(DroolsProperties.PROPERTY_CONTROLLER_NAME);
+        if (controllerPropName == null) {
+            controllerProperties.setProperty(DroolsProperties.PROPERTY_CONTROLLER_NAME, controllerName);
+        } else if (!controllerPropName.equals(controllerName)) {
+            logger.error("{}: mismatch controller named {} against {} in file {}",
+                         this, controllerPropName, controllerName, controllerFilename);
+            return false;
+        }
+        return true;
+    }
+
+
+    @Override
     public boolean backupController(String controllerName) {
         final Path controllerPropertiesPath = Paths.get(this.configurationDirectory.toString(),
                 controllerName + PROPERTIES_FILE_CONTROLLER_SUFFIX);
@@ -209,97 +325,6 @@ public class FileSystemPersistence implements SystemPersistence {
         return true;
     }
 
-    @Override
-    public Properties getControllerProperties(String controllerName) {
-        return this.getProperties(controllerName + CONTROLLER_SUFFIX_IDENTIFIER);
-    }
-
-    @Override
-    public List<Properties> getControllerProperties() {
-        final List<Properties> controllers = new ArrayList<>();
-        final File[] controllerFiles = this.sortedListFiles();
-        for (final File controllerFile : controllerFiles) {
-            if (controllerFile.getName().endsWith(PROPERTIES_FILE_CONTROLLER_SUFFIX)) {
-                final int idxSuffix = controllerFile.getName().indexOf(PROPERTIES_FILE_CONTROLLER_SUFFIX);
-                final int lastIdxSuffix =
-                        controllerFile.getName().lastIndexOf(PROPERTIES_FILE_CONTROLLER_SUFFIX);
-                if (idxSuffix != lastIdxSuffix) {
-                    throw new IllegalArgumentException(
-                            "Improper naming of controller properties file: " + "Expected <controller-name>"
-                                    + FileSystemPersistence.PROPERTIES_FILE_CONTROLLER_SUFFIX);
-                }
-
-                final String name = controllerFile.getName().substring(0, lastIdxSuffix);
-                try {
-                    final Properties controllerProperties = this.getControllerProperties(name);
-                    final String controllerName =
-                            controllerProperties.getProperty(DroolsProperties.PROPERTY_CONTROLLER_NAME);
-                    if (controllerName == null) {
-                        controllerProperties.setProperty(DroolsProperties.PROPERTY_CONTROLLER_NAME, name);
-                    } else if (!controllerName.equals(name)) {
-                        logger.error("{}: mismatch controller named {} with file name {}", this, controllerName,
-                                controllerFile.getName());
-                        continue;
-                    }
-                    controllers.add(this.getControllerProperties(name));
-                } catch (final Exception e) {
-                    logger.error("{}: cannot obtain properties for controller {} because of {}", name,
-                            e.getMessage(), e);
-                }
-            }
-        }
-        return controllers;
-    }
-
-    @Override
-    public Properties getProperties(String name) {
-        final Path propertiesPath =
-                Paths.get(this.configurationDirectory.toString(), name + ".properties");
-
-        if (!Files.exists(propertiesPath)) {
-            throw new IllegalArgumentException("properties for " + name + " are not persisted.");
-        }
-
-        try {
-            return PropertyUtil.getProperties(propertiesPath.toFile());
-        } catch (final Exception e) {
-            logger.warn("{}: can't read properties @ {}", name, propertiesPath);
-            throw new IllegalArgumentException(
-                    "can't read properties for " + name + " @ " + propertiesPath, e);
-        }
-    }
-
-    @Override
-    public List<Properties> getEnvironmentProperties() {
-        final List<Properties> envs = new ArrayList<>();
-        final File[] envFiles = this.sortedListFiles();
-        for (final File envFile : envFiles) {
-            if (envFile.getName().endsWith(ENV_SUFFIX)) {
-                final String name = envFile.getName().substring(0, envFile.getName().indexOf(ENV_SUFFIX));
-                try {
-                    envs.add(this.getEnvironmentProperties(name));
-                } catch (final Exception e) {
-                    logger.error("{}: cannot get environment {} because of {}", name, e.getMessage(), e);
-                }
-            }
-        }
-        return envs;
-    }
-
-    @Override
-    public Properties getEnvironmentProperties(String name) {
-        final Path envPath = Paths.get(this.configurationDirectory.toString(), name + ENV_SUFFIX);
-        if (!Files.exists(envPath)) {
-            throw new IllegalArgumentException("{} environment" + name + " cannot be retrieved");
-        }
-
-        try {
-            return PropertyUtil.getProperties(envPath.toFile());
-        } catch (final Exception e) {
-            throw new IllegalArgumentException("cannot read environment " + name, e);
-        }
-    }
-
     /**
      * provides a list of files sorted by name in ascending order in the configuration directory.
      */
diff --git a/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java b/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
index 34b27c23..32852225 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/persistence/SystemPersistence.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * policy-management
  * ================================================================================
- * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,27 +31,32 @@ public interface SystemPersistence {
     /**
      * configuration directory.
      */
-    public static final String DEFAULT_CONFIGURATION_DIR = "config";
+    String DEFAULT_CONFIGURATION_DIR = "config";
 
     /**
      * Persistence Manager. For now it is a file-based properties management, In the future, it will
      * probably be DB based, so manager implementation will change.
      */
-    public static final SystemPersistence manager = new FileSystemPersistence();
+    SystemPersistence manager = new FileSystemPersistence();
 
     /**
      * sets a configuration directory and ensures it exists.
      *
      * @param configDir configuration directory or null to use the default one
      */
-    public void setConfigurationDir(String configDir);
+    void setConfigurationDir(String configDir);
+
+    /**
+     * sets the default configuration directory and ensures it exists.
+     */
+    void setConfigurationDir();
 
     /**
      * get configuration directory path.
      *
      * @return configuration directory path
      */
-    public Path getConfigurationPath();
+    Path getConfigurationPath();
 
     /**
      * backs up a controller configuration.
@@ -59,7 +64,7 @@ public interface SystemPersistence {
      * @param controllerName the controller name
      * @return true if the configuration is backed up
      */
-    public boolean backupController(String controllerName);
+    boolean backupController(String controllerName);
 
     /**
      * persists controller configuration.
@@ -71,7 +76,7 @@ public interface SystemPersistence {
      * @throws IllegalArgumentException if the configuration cannot be handled by the persistence
      *         manager
      */
-    public boolean storeController(String controllerName, Object configuration);
+    boolean storeController(String controllerName, Object configuration);
 
     /**
      * delete controller configuration.
@@ -79,7 +84,7 @@ public interface SystemPersistence {
      * @param controllerName the controller name
      * @return true if storage is succesful, false otherwise
      */
-    public boolean deleteController(String controllerName);
+    boolean deleteController(String controllerName);
 
     /**
      * get controller properties.
@@ -90,34 +95,34 @@ public interface SystemPersistence {
      * @throws IllegalArgumentException if the controller name does not lead to a properties
      *         configuration
      */
-    public Properties getControllerProperties(String controllerName);
+    Properties getControllerProperties(String controllerName);
 
     /**
      * get controllers configuration.
      *
      * @return list of controllers properties
      */
-    public List<Properties> getControllerProperties();
+    List<Properties> getControllerProperties();
 
     /**
      * get environments.
      *
      */
-    public List<Properties> getEnvironmentProperties();
+    List<Properties> getEnvironmentProperties();
 
     /**
      * get environment properties.
      *
      * @param environmentName name
      */
-    public Properties getEnvironmentProperties(String environmentName);
+    Properties getEnvironmentProperties(String environmentName);
 
     /**
      * get the engine properties.
      *
      * @return the engine properties
      */
-    public Properties getEngineProperties();
+    Properties getEngineProperties();
 
     /**
      * get properties by name.
@@ -127,5 +132,20 @@ public interface SystemPersistence {
      *
      * @throws IllegalArgumentException if the name does not lead to a properties configuration
      */
-    public Properties getProperties(String name);
+    Properties getProperties(String name);
+
+    /**
+     * get system properties configuration.
+     *
+     * @param name name
+     * @return properties
+     */
+    Properties getSystemProperties(String name);
+
+    /**
+     * get system properties configuration list.
+     *
+     * @return list of system properties
+     */
+    List<Properties> getSystemProperties();
 }
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/Main.java b/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
index 79268b2c..5ebd0490 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/Main.java
@@ -1,8 +1,8 @@
 /*-
  * ============LICENSE_START=======================================================
- * policy-management
+ * ONAP
  * ================================================================================
- * Copyright (C) 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,12 +20,10 @@
 
 package org.onap.policy.drools.system;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
 import java.util.Properties;
-
 import org.onap.policy.drools.persistence.SystemPersistence;
 import org.onap.policy.drools.properties.DroolsProperties;
+import org.onap.policy.drools.utils.PropertyUtil;
 import org.onap.policy.drools.utils.logging.LoggerUtil;
 import org.onap.policy.drools.utils.logging.MDCTransaction;
 import org.slf4j.Logger;
@@ -36,13 +34,6 @@ import org.slf4j.LoggerFactory;
  */
 public class Main {
 
-    /** logback configuration file system property. */
-    public static final String LOGBACK_CONFIGURATION_FILE_SYSTEM_PROPERTY =
-            "logback.configurationFile";
-
-    /** logback configuration file system property. */
-    public static final String LOGBACK_CONFIGURATION_FILE_DEFAULT = "config/logback.xml";
-
     /** constructor (hides public default one). */
     private Main() {}
 
@@ -53,44 +44,27 @@ public class Main {
      */
     public static void main(String[] args) {
 
-        /* logging defaults */
-
-        if (System.getProperty(LOGBACK_CONFIGURATION_FILE_SYSTEM_PROPERTY) == null) {
-            if (Files.exists(Paths.get(LOGBACK_CONFIGURATION_FILE_DEFAULT))) {
-                System.setProperty(
-                        LOGBACK_CONFIGURATION_FILE_SYSTEM_PROPERTY, LOGBACK_CONFIGURATION_FILE_DEFAULT);
-            } else {
-                LoggerUtil.setLevel(LoggerUtil.ROOT_LOGGER, ch.qos.logback.classic.Level.INFO.toString());
-            }
-        }
-
-        /* make sure the default configuration directory is properly set up */
+        /* start logger */
 
-        SystemPersistence.manager.setConfigurationDir(null);
+        Logger logger = LoggerFactory.getLogger(Main.class);
 
-        /* logging defaults */
+        /* system properties */
 
-        if (System.getProperty(LOGBACK_CONFIGURATION_FILE_SYSTEM_PROPERTY) == null) {
-            if (Files.exists(Paths.get(LOGBACK_CONFIGURATION_FILE_DEFAULT))) {
-                System.setProperty(
-                        LOGBACK_CONFIGURATION_FILE_SYSTEM_PROPERTY, LOGBACK_CONFIGURATION_FILE_DEFAULT);
-            } else {
-                LoggerUtil.setLevel(LoggerUtil.ROOT_LOGGER, ch.qos.logback.classic.Level.INFO.toString());
-            }
+        for (Properties systemProperties : SystemPersistence.manager.getSystemProperties()) {
+            PropertyUtil.setSystemProperties(systemProperties);
         }
 
         /* 0. boot */
 
         PolicyEngine.manager.boot(args);
 
-        /* start logger */
+        /* 1.a. Configure Engine */
 
-        final Logger logger = LoggerFactory.getLogger(Main.class);
-
-        /* 1.a. Configure the Engine */
-
-        Properties engineProperties = SystemPersistence.manager.getEngineProperties();
-        if (engineProperties == null) {
+        Properties engineProperties;
+        try {
+            engineProperties = SystemPersistence.manager.getEngineProperties();
+        } catch (IllegalArgumentException iae) {
+            logger.warn("Main: engine properties not found.  Using default configuration.", iae);
             engineProperties = PolicyEngine.manager.defaultTelemetryConfig();
         }
 
@@ -98,7 +72,7 @@ public class Main {
 
         /* 1.b. Load Installation Environment(s) */
 
-        for (final Properties env : SystemPersistence.manager.getEnvironmentProperties()) {
+        for (Properties env : SystemPersistence.manager.getEnvironmentProperties()) {
             PolicyEngine.manager.setEnvironment(env);
         }
 
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
index 959114a2..bedd8d48 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
@@ -54,6 +54,7 @@ import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
 import org.onap.policy.drools.server.restful.RestManager;
 import org.onap.policy.drools.server.restful.aaf.AafTelemetryAuthFilter;
+import org.onap.policy.drools.utils.PropertyUtil;
 import org.onap.policy.drools.utils.logging.LoggerUtil;
 import org.onap.policy.drools.utils.logging.MDCTransaction;
 import org.slf4j.Logger;
@@ -429,7 +430,7 @@ class PolicyEngineManager implements PolicyEngine {
 
     @Override
     public synchronized void setEnvironment(Properties properties) {
-        this.environment.putAll(properties);
+        this.environment.putAll(PropertyUtil.getInterpolatedProperties(properties));
     }
 
     @JsonIgnore
@@ -443,7 +444,10 @@ class PolicyEngineManager implements PolicyEngine {
     public synchronized String getEnvironmentProperty(String envKey) {
         String value = this.environment.getProperty(envKey);
         if (value == null) {
-            value = System.getenv(envKey);
+            value = System.getProperty(envKey);
+            if (value == null) {
+                value = System.getenv(envKey);
+            }
         }
         return value;
     }