support local config and policy files

* DCAEGEN2-2733/DCAEGEN2-2753 - onap-dcae-cbs-docker-client support local config and policy files
	Support $CBS_CLIENT_CONFIG_PATH and $CBS_CLIENT_POLICY_PATH,
	Default paths are /app-config/application_config.yaml and /etc/policies/policies.json.
	If files are missing or malformed, drop back to the current behavior.

Change-Id: Iba2484f6be7fc54e412f7de302bb79a4d7bfde17
Signed-off-by: Hansen, Tony (th1395) <th1395@att.com>
Issue-ID: DCAEGEN2-2733
Issue-ID: DCAEGEN2-2753
Signed-off-by: Hansen, Tony (th1395) <th1395@att.com>

2 files changed, 129 insertions, 14 deletions

diff --git a/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/client.py b/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/client.py
index 7f36169..c42b4ae 100644
--- a/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/client.py
+++ b/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/client.py
@@ -1,5 +1,5 @@
 # ================================================================================
-# Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2017-2021 AT&T Intellectual Property. All rights reserved.
 # Copyright (C) 2021 Nokia. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,18 +14,36 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============LICENSE_END=========================================================
-#
-# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+""" provide a means to return a configuration to CBS """
 
 import json
 import os
 import requests
+import sys
+import yaml
+
 from onap_dcae_cbs_docker_client import get_module_logger
 from onap_dcae_cbs_docker_client.exceptions import ENVsMissing, CantGetConfig, CBSUnreachable
 
 logger = get_module_logger(__name__)
 
 
+DEFAULT_CONFIG_PATH = "/app-config/application_config.yaml"
+DEFAULT_POLICY_PATH = "/etc/policies/policies.json"
+
+
+# provide a means to import the default paths into unit tests
+# For some reason, tox does not like
+# from onap_dcae_cbs_docker_client.client import DEFAULT_CONFIG_PATH, DEFAULT_POLICY_PATH
+
+def default_config_path():
+    return DEFAULT_CONFIG_PATH
+
+
+def default_policy_path():
+    return DEFAULT_POLICY_PATH
+
 #########
 # HELPERS
 
@@ -35,7 +53,7 @@ def _recurse(config):
     Recurse through a configuration, or recursively a sub element of it.
     If it's a dict: recurse over all the values
     If it's a list: recurse over all the values
-    If it's a string: return the replacement
+    If it's a string: expand the string and return its replacement
     If none of the above, just return the item.
     """
     if isinstance(config, list):
@@ -96,11 +114,13 @@ def change_envs(value):
     """
     Replace env reference by actual value and return it
     """
-    if value.startswith('$'):
-        try:
-            value = os.environ[value.replace('${', '').replace('}', '')]
-        except KeyError as e:
-            raise ENVsMissing("Required ENV Variable {0} missing".format(e))
+    if value.startswith('${'):
+        name = value[2:-1]
+        if name in os.environ:
+            return os.environ[name]
+
+        logger.error(f"Required ENV Variable '{name}' missing. Is '{value}' properly formatted?")
+        raise ENVsMissing(f"Required ENV Variable {name} missing. Is '{value}' properly formatted?")
     return value
 
 
@@ -108,18 +128,113 @@ def change_envs(value):
 # Public
 def get_all():
     """
-    Hit the CBS service_component_all endpoint
+    If not configured locally,
+    hit the CBS service_component_all endpoint
+
+    Local configuration comes from $CBS_CLIENT_CONFIG_PATH and $CBS_CLIENT_POLICY_PATH,
+    defaulted to /app-config/application_config.yaml and /etc/policies/policies.json.
     """
+    config_path = os.getenv("CBS_CLIENT_CONFIG_PATH", DEFAULT_CONFIG_PATH)
+    if config_path == "":
+        config_path = DEFAULT_CONFIG_PATH
+    policy_path = os.getenv("CBS_CLIENT_POLICY_PATH", DEFAULT_POLICY_PATH)
+    if policy_path == "":
+        policy_path = DEFAULT_POLICY_PATH
+
+    try:
+        logger.debug(f"opening config_path={config_path}")
+        with open(config_path) as fp:
+            config = yaml.safe_load(fp)
+
+        policies = None
+        try:
+            logger.debug(f"opening policy_path={policy_path}")
+            with open(policy_path) as fp:
+                policies = json.load(fp)
+
+        except FileNotFoundError:
+            logger.debug("Policy File Not Found exception received")
+            pass
+
+        except (json.decoder.JSONDecodeError, ValueError) as e:
+            logger.error(f"The policy file '{policy_path}' has invalid JSON: %s", e)
+            pass
+
+        except Exception as e:
+            logger.error(f"An error occurred processing the policy file '{policy_path}': %s", e)
+            import traceback
+            traceback.print_exc(file=sys.stderr)
+            pass
+
+        if policies is not None:
+            if "policies" in policies:
+                logger.debug(f"Returning config read from {config_path} an policy read from {policy_path}")
+                ret = {"config": _recurse(config), "policies": policies["policies"]}
+            else:
+                logger.error(f"The policy file '{policy_path}' does NOT have a 'policies' block in it.")
+                ret = {"config": _recurse(config)}
+        else:
+            logger.debug(f"Returning config read from {config_path}")
+            ret = {"config": _recurse(config)}
+
+        return ret
+
+    except yaml.scanner.ScannerError as e:
+        logger.error(f"The configuration file '{config_path}' has invalid YAML: {e}")
+        pass
+
+    except FileNotFoundError:
+        logger.debug("Config File Not Found exception received")
+        pass
+
+    except Exception as e:
+        logger.error(f"An error occurred processing the configuration file '{config_path}': {e}")
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        pass
+
+    logger.debug("Fallback to using REST call")
     config = _get_path("service_component_all")
     return _recurse(config)
 
 
 def get_config():
     """
-    Hit the CBS service_component endpoint
+    If not configured locally,
+    hit the CBS service_component endpoint for the configuration.
+
+    Local configuration comes from $CBS_CLIENT_CONFIG_PATH,
+    defaulted to /app-config/application_config.yaml.
 
     TODO: should we take in a "retry" boolean, and retry on behalf of the caller?
     Currently, we return an exception and let the application decide how it wants to proceed (Crash, try again, etc).
     """
+    config_path = os.getenv("CBS_CLIENT_CONFIG_PATH", DEFAULT_CONFIG_PATH)
+    if config_path == "":
+        config_path = DEFAULT_CONFIG_PATH
+
+    try:
+        logger.debug(f"opening config_path={config_path}")
+        with open(config_path) as fp:
+            config = yaml.safe_load(fp)
+
+        logger.debug(f"Returning config read from {config_path}")
+        return _recurse({"config": config})
+
+    except yaml.scanner.ScannerError as e:
+        logger.error(f"The configuration file '{config_path}' has invalid YAML: {e}")
+        pass
+
+    except FileNotFoundError:
+        logger.debug("Config File Not Found exception received")
+        pass
+
+    except Exception as e:
+        logger.error(f"An error occurred processing the configuration file '{config_path}': {e}")
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        pass
+
+    logger.debug("Fallback to using REST call")
     config = _get_path("service_component")
     return _recurse(config)
diff --git a/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/exceptions.py b/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/exceptions.py
index 6b73b91..0cd19fd 100644
--- a/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/exceptions.py
+++ b/onap-dcae-cbs-docker-client/onap_dcae_cbs_docker_client/exceptions.py
@@ -1,5 +1,5 @@
 # ================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2019-2021 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============LICENSE_END=========================================================
-#
-# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+""" exceptions used by the Config Binding Server """
 
 
 class ENVsMissing(Exception):