diff options
Diffstat (limited to 'dcae-cli/dcae_cli/util')
| -rw-r--r-- | dcae-cli/dcae_cli/util/config.py | 2 | ||||
| -rw-r--r-- | dcae-cli/dcae_cli/util/discovery.py | 187 | ||||
| -rw-r--r-- | dcae-cli/dcae_cli/util/docker_util.py | 16 | ||||
| -rw-r--r-- | dcae-cli/dcae_cli/util/policy.py | 64 | ||||
| -rw-r--r-- | dcae-cli/dcae_cli/util/run.py | 4 |
5 files changed, 265 insertions, 8 deletions
diff --git a/dcae-cli/dcae_cli/util/config.py b/dcae-cli/dcae_cli/util/config.py index c9df69a..7444fb1 100644 --- a/dcae-cli/dcae_cli/util/config.py +++ b/dcae-cli/dcae_cli/util/config.py @@ -124,7 +124,7 @@ def get_docker_logins_key(): def get_path_component_spec(): return get_config().get("path_component_spec", - "/component-json-schemas/component-specification/dcae-cli-v1/component-spec-schema.json") + "/component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json") def get_path_data_format(): return get_config().get("path_data_format", diff --git a/dcae-cli/dcae_cli/util/discovery.py b/dcae-cli/dcae_cli/util/discovery.py index ba74f1f..0fc0165 100644 --- a/dcae-cli/dcae_cli/util/discovery.py +++ b/dcae-cli/dcae_cli/util/discovery.py @@ -28,6 +28,7 @@ import contextlib from collections import defaultdict from itertools import chain from functools import partial +from datetime import datetime from uuid import uuid4 import six @@ -39,6 +40,8 @@ from dcae_cli.util.exc import DcaeException from dcae_cli.util.profiles import get_profile from dcae_cli.util.config import get_docker_logins_key +import os +import click logger = get_logger('Discovery') @@ -342,6 +345,12 @@ def _create_dmaap_key(config_key): return "{:}:dmaap".format(config_key) +def _create_policies_key(config_key): + """Create policies key from config key + + Assumes config_key is well-formed""" + return "{:}:policies/".format(config_key) + def clear_user_instances(user, host=None): '''Removes all Consul key:value entries for a given user''' host = _choose_consul_host(host) @@ -510,6 +519,11 @@ def push_config(conf_key, conf, rels_key, rels, dmaap_key, dmaap_map, host=None) for k, v in ((conf_key, conf), (rels_key, rels), (dmaap_key, dmaap_map)): cons.kv.put(k, json.dumps(v)) + logger.info("* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *") + logger.info("* If you run a 'component reconfig' command, you must first execute the following") + logger.info("* export SERVICE_NAME={:}".format(conf_key)) + logger.info("* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *") + def remove_config(config_key, host=None): """Deletes a config from Consul @@ -520,9 +534,10 @@ def remove_config(config_key, host=None): """ host = _choose_consul_host(host) cons = Consul(host) - results = [ cons.kv.delete(k) for k in (config_key, _create_rels_key(config_key), \ - _create_dmaap_key(config_key)) ] - return all(results) + # "recurse=True" deletes the SERVICE_NAME KV and all other KVs with suffixes (:rel, :dmaap, :policies) + results = cons.kv.delete(config_key, recurse=True) + + return results def _group_config(config, config_key_map): @@ -559,7 +574,8 @@ def config_context(user, cname, cver, params, interface_map, instance_map, Args ---- - always_cleanup: (boolean) This context manager will cleanup the produced config + always_cleanup: (boolean) + This context manager will cleanup the produced config context always if this is True. When False, cleanup will only occur upon any exception getting thrown in the context manager block. Default is True. force: (boolean) @@ -596,3 +612,166 @@ def config_context(user, cname, cver, params, interface_map, instance_map, pass else: remove_config(conf_key, host) + + +def policy_update(policy_change_file): + + # Determine if it is an 'updated_policies' or 'removed_policies' change, or if user included ALL policies + policies = True if "policies" in policy_change_file.keys() else False + updated = True if "updated_policies" in policy_change_file.keys() else False + removed = True if "removed_policies" in policy_change_file.keys() else False + + cons = Consul(consul_host) + service_name = os.environ["SERVICE_NAME"] + policy_folder = service_name + ":policies/items/" + event_folder = service_name + ":policies/event" + + if policies: + # User specified ALL "policies" in the Policy File. Ignore "updated_policies"/"removed_policies" + logger.warning("The 'policies' specified in the 'policy-file' will replace all policies in Consul.") + allPolicies = policy_change_file['policies'] + if not update_all_policies(cons, policy_folder, allPolicies): + return False + + else: + # If 'removed_policies', delete the Policy from the Component KV pair + if removed: + policyDeletes = policy_change_file['removed_policies'] + if not remove_policies(cons, policy_folder, policyDeletes): + return False + + # If 'updated_policies', update the Component KV pair + if updated: + policyUpdates = policy_change_file['updated_policies'] + if not update_specified_policies(cons, policy_folder, policyUpdates): + return False + + return create_policy_event(cons, event_folder, policy_folder) + + +def create_policy_event(cons, event_folder, policy_folder): + """ Create a Policy 'event' KV pair in Consol """ + + timestamp = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") + update_id = str(uuid4()) + policies = cons.kv.get(policy_folder, recurse=True) + policies_count = str(policies).count("'Key':") + + event = '{"action": "gathered", "timestamp": "' + timestamp + '", "update_id": "' + update_id + '", "policies_count": ' + str(policies_count) + '}' + if not cons.kv.put(event_folder, event): + logger.error("Policy 'Event' creation of ({:}) in Consul failed".format(event_folder)) + return False + + return True + + +def update_all_policies(cons, policy_folder, allPolicies): + """ Delete all policies from Consul, then add the policies the user specified in the 'policies' section of the policy-file """ + + if not cons.kv.delete(policy_folder, recurse=True): # Deletes all Policies under the /policies/items folder + logger.error("Policy delete of ({:}) in Consul failed".format(policy_folder)) + return False + + if not update_specified_policies(cons, policy_folder, allPolicies): + return False + + return True + +def update_specified_policies(cons, policy_folder, policyUpdates): + """ Replace the policies the user specified in the 'updated_policies' (or 'policies') section of the policy-file """ + + for policy in policyUpdates: + policy_folder_id = extract_policy_id(policy_folder, policy) + if policy_folder_id: + policyBody = json.dumps(policy) + if not cons.kv.put(policy_folder_id, policyBody): + logger.error("Policy update of ({:}) in Consul failed".format(policy_folder_id)) + return False + else: + return False + + return True + + +def remove_policies(cons, policy_folder, policyDeletes): + """ Delete the policies that the user specified in the 'removed_policies' section of the policy-file """ + + for policy in policyDeletes: + policy_folder_id = extract_policy_id(policy_folder, policy) + if policy_folder_id: + if not cons.kv.delete(policy_folder_id): + logger.error("Policy delete of ({:}) in Consul failed".format(policy_folder_id)) + return False + else: + return False + + return True + +def extract_policy_id(policy_folder, policy): + """ Extract the Policy ID from the policyName. + Return the Consul key (Policy Folder with Policy ID) """ + + policyId_re = re.compile(r"(.*)\.\d+\.[a-zA-Z]+$") + + policyName = policy['policyName'] # Extract the policy Id "Consul Key" from the policy name + match = policyId_re.match(policyName) + + if match: + policy_id = match.group(1) + policy_folder_id = policy_folder + policy_id + + return policy_folder_id + else: + logger.error("policyName ({:}) needs to end in '.#.xml' in order to extract the Policy ID".format(policyName)) + return + + +def build_policy_command(policy_reconfig_path, policy_change_file): + """ Build command to execute the Policy Reconfig script in the Docker container """ + + # Determine if it is an 'updated_policies' and/or 'removed_policies' change, or if user included ALL policies + all_policies = True if "policies" in policy_change_file.keys() else False + updated = True if "updated_policies" in policy_change_file.keys() else False + removed = True if "removed_policies" in policy_change_file.keys() else False + + # Create the Reconfig Script command (3 parts: Command and 2 ARGs) + command = [] + command.append(policy_reconfig_path) + command.append("policies") + + # Create a Dictionary of 'updated', 'removed', and 'ALL' policies + + # 'updated' policies - policies come from the --policy-file + if updated: + updated_policies = policy_change_file['updated_policies'] + else: updated_policies = [] + + policies = {} + policies["updated_policies"] = updated_policies + + # 'removed' policies - policies come from the --policy-file + if removed: + removed_policies = policy_change_file['removed_policies'] + else: removed_policies = [] + + policies["removed_policies"] = removed_policies + + # ALL 'policies' - policies come from Consul + cons = Consul(consul_host) + service_name = os.environ["SERVICE_NAME"] + policy_folder = service_name + ":policies/items/" + + id, consul_policies = cons.kv.get(policy_folder, recurse=True) + + policy_values = [] + if consul_policies: + for policy in consul_policies: + policy_value = json.loads(policy['Value']) + policy_values.append(policy_value) + + policies["policies"] = policy_values + + # Add the policies to the Docker "command" as a JSON string + command.append(json.dumps(policies)) + + return command diff --git a/dcae-cli/dcae_cli/util/docker_util.py b/dcae-cli/dcae_cli/util/docker_util.py index 7ae933f..3e29f5c 100644 --- a/dcae-cli/dcae_cli/util/docker_util.py +++ b/dcae-cli/dcae_cli/util/docker_util.py @@ -32,7 +32,6 @@ import dockering as doc from dcae_cli.util.logger import get_logger from dcae_cli.util.exc import DcaeException - dlog = get_logger('Docker') _reg_img = 'gliderlabs/registrator:latest' @@ -188,6 +187,7 @@ def deploy_component(profile, image, instance_name, docker_config, should_wait=F client = get_docker_client(profile, logins=logins) config = doc.create_container_config(client, image, envs, hcp) + return _run_container(client, config, name=instance_name, wait=should_wait) @@ -210,3 +210,17 @@ def undeploy_component(client, image, instance_name): except Exception as e: dlog.error("Error while undeploying Docker container/image: {0}".format(e)) return False + +def reconfigure(client, instance_name, command): + """ Execute the Reconfig script in the Docker container """ + + # 'command' has 3 parts in a list (1 Command and 2 ARGs) + exec_Id = client.exec_create(container=instance_name, cmd=command) + + exec_start_resp = client.exec_start(exec_Id, stream=True) + + # Using a 'single' generator response to solve issue of 'start_exec' returning control after 6 minutes + for response in exec_start_resp: + dlog.info("Reconfig Script execution response: {:}".format(response)) + exec_start_resp.close() + break diff --git a/dcae-cli/dcae_cli/util/policy.py b/dcae-cli/dcae_cli/util/policy.py new file mode 100644 index 0000000..2da9f0b --- /dev/null +++ b/dcae-cli/dcae_cli/util/policy.py @@ -0,0 +1,64 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +""" +Function for Policy schema validation +""" + +from jsonschema import validate, ValidationError +from dcae_cli.util.logger import get_logger +from dcae_cli.util import reraise_with_msg + +logger = get_logger('policy') + +_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Schema for policy changes", + "type": "object", + "properties": { + "updated_policies": {"type": "array"}, + "removed_policies": {"type": "array"}, + "policies": {"type": "array"} + }, + "additionalProperties": False +} + +_validation_msg = """ +Is your Policy file a valid json? +Does your Policy file follow this format? + +{ + "updated_policies": [{},{},...], + "removed_policies": [{},{},...], + "policies": [{},{},...] +} +""" + + +def validate_against_policy_schema(policy_file): + """Validate the policy file against the schema""" + + try: + validate(policy_file, _SCHEMA) + except ValidationError as e: + logger.error("Policy file validation issue") + logger.error(_validation_msg) + reraise_with_msg(e, as_dcae=True) +
\ No newline at end of file diff --git a/dcae-cli/dcae_cli/util/run.py b/dcae-cli/dcae_cli/util/run.py index f0f1309..293c725 100644 --- a/dcae-cli/dcae_cli/util/run.py +++ b/dcae-cli/dcae_cli/util/run.py @@ -130,8 +130,8 @@ def run_component(user, cname, cver, catalog, additional_user, attached, force, Args ---- force: (boolean) - Continue to run even when there are no valid downstream components when - this flag is set to True. + Continue to run even when there are no valid downstream components, + when this flag is set to True. dmaap_map: (dict) config_key to message router or data router connections. Used as a manual way to make available this information for the component. inputs_map: (dict) config_key to value that is intended to be provided at |