From 01a60ff23b979eb676658713748598ba4892163a Mon Sep 17 00:00:00 2001 From: Miroslav Los Date: Tue, 26 Nov 2019 14:20:36 +0100 Subject: Support python3 in all plugins Unify tox/requirements/setup.py requirement specifications. Do not set upper version limits if possible. Drop uuid as dependency included with standard library. Drop import of unmaintained cloudify_importer without python3 version. Use PEP 508 URLs in requirements for non-PyPI (github) releases. Use cloudify-common 5 release; pre-release package for python3. Rewrite uses of map with loops/comprehensions. Signed-off-by: Miroslav Los Issue-ID: DCAEGEN2-1956 Change-Id: I7b3ceb97a628e3af5bda3178d182f4207069e86d --- cdap/cdapplugin/requirements.txt | 3 +- cdap/cdapplugin/setup.py | 6 ++-- cdap/cdapplugin/tox-local.ini | 9 ++--- cdap/cdapplugin/tox.ini | 8 ++--- clamp-policy/requirements.txt | 4 ++- clamp-policy/setup.py | 4 ++- clamp-policy/tox-local.ini | 8 ++--- clamp-policy/tox.ini | 10 ++---- dcae-policy/dcaepolicyplugin/tasks.py | 15 ++++++--- dcae-policy/requirements.txt | 4 ++- dcae-policy/setup.py | 4 ++- dcae-policy/tests/test_discovery.py | 3 +- dcae-policy/tox-local.ini | 11 ++++--- dcae-policy/tox.ini | 13 ++++---- docker/dockerplugin/discovery.py | 12 +++---- docker/dockerplugin/tasks.py | 47 +++++++++++++-------------- docker/dockerplugin/utils.py | 3 +- docker/requirements.txt | 7 ++-- docker/setup.py | 9 ++--- docker/tox.ini | 9 ++--- k8s/configure/configure.py | 8 +++-- k8s/k8sclient/k8sclient.py | 11 +++---- k8s/k8splugin/discovery.py | 9 +++-- k8s/k8splugin/tasks.py | 40 ++++++++++------------- k8s/k8splugin/utils.py | 3 +- k8s/requirements.txt | 10 +++--- k8s/setup.py | 11 +++---- k8s/tests/test_k8sclient.py | 7 ++-- k8s/tox.ini | 8 ++--- relationships/relationshipplugin/discovery.py | 6 +++- relationships/requirements.txt | 4 ++- relationships/setup.py | 4 ++- relationships/tox.ini | 10 +++--- 33 files changed, 164 insertions(+), 156 deletions(-) diff --git a/cdap/cdapplugin/requirements.txt b/cdap/cdapplugin/requirements.txt index ff93d4c..43a0ea1 100644 --- a/cdap/cdapplugin/requirements.txt +++ b/cdap/cdapplugin/requirements.txt @@ -1,2 +1,3 @@ onap-dcae-dcaepolicy-lib==1.0.0 -uuid==1.30 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/cdap/cdapplugin/setup.py b/cdap/cdapplugin/setup.py index 6ca9461..978af27 100644 --- a/cdap/cdapplugin/setup.py +++ b/cdap/cdapplugin/setup.py @@ -1,6 +1,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -32,7 +33,8 @@ setup( url = "https://gerrit.onap.org/r/#/admin/projects/dcaegen2/platform/plugins", zip_safe=False, install_requires = [ - "onap-dcae-dcaepolicy-lib>=1.0.0,<2.0.0" - "uuid==1.30" + # FIXME: not compatible with latest version + 'onap-dcae-dcaepolicy-lib==1.0.0', + 'cloudify-common>=5.0.0', ] ) diff --git a/cdap/cdapplugin/tox-local.ini b/cdap/cdapplugin/tox-local.ini index d553704..1664561 100644 --- a/cdap/cdapplugin/tox-local.ini +++ b/cdap/cdapplugin/tox-local.ini @@ -1,10 +1,11 @@ [tox] -envlist = py27 +envlist = py27,py36 + [testenv] deps= - -rrequirements.txt - cloudify==3.4 + -rrequirements.txt pytest coverage pytest-cov -commands=pytest --cov {envsitepackagesdir}/cdapcloudify --cov-report html +commands= + pytest --cov {envsitepackagesdir}/cdapcloudify --cov-report html diff --git a/cdap/cdapplugin/tox.ini b/cdap/cdapplugin/tox.ini index 2f4bb03..32445ed 100644 --- a/cdap/cdapplugin/tox.ini +++ b/cdap/cdapplugin/tox.ini @@ -1,14 +1,12 @@ [tox] -envlist = py27 +envlist = py27,py36 + [testenv] deps= - -rrequirements.txt - cloudify==3.4 + -rrequirements.txt pytest coverage pytest-cov -setenv= - PYTHONPATH={toxinidir} commands= pytest --junitxml xunit-results.xml --cov cdapcloudify --cov-report xml coverage xml diff --git a/clamp-policy/requirements.txt b/clamp-policy/requirements.txt index 025de9d..e9526b8 100644 --- a/clamp-policy/requirements.txt +++ b/clamp-policy/requirements.txt @@ -1 +1,3 @@ -requests>=2.11.0,<3.0.0 \ No newline at end of file +requests>=2.11.0 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/clamp-policy/setup.py b/clamp-policy/setup.py index de6b119..25ad51b 100644 --- a/clamp-policy/setup.py +++ b/clamp-policy/setup.py @@ -1,5 +1,6 @@ # ================================================================================ # Copyright (c) 2019 Wipro Limited Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +27,8 @@ setup( author='Vignesh K', packages=['clamppolicyplugin'], install_requires=[ - "requests>=2.11.0,<3.0.0" + 'requests>=2.11.0', + 'cloudify-common>=5.0.0', ], keywords='clamp policy model cloudify plugin', classifiers=[ diff --git a/clamp-policy/tox-local.ini b/clamp-policy/tox-local.ini index ad84024..e41b3f4 100644 --- a/clamp-policy/tox-local.ini +++ b/clamp-policy/tox-local.ini @@ -1,15 +1,13 @@ # tox -c tox-local.ini | tee -a logs/test_clamppolicyplugin.log 2>&1 [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= -rrequirements.txt - cloudify-plugins-common==3.4 pytest coverage pytest-cov -setenv = - PYTHONPATH={toxinidir} # recreate = True -commands=pytest -v --cov clamppolicyplugin --cov-report html +commands= + pytest -v --cov clamppolicyplugin --cov-report html diff --git a/clamp-policy/tox.ini b/clamp-policy/tox.ini index 3974a7e..9e09c4a 100644 --- a/clamp-policy/tox.ini +++ b/clamp-policy/tox.ini @@ -1,17 +1,13 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= -rrequirements.txt - cloudify-plugins-common==3.4 pytest coverage pytest-cov -setenv = - PYTHONPATH={toxinidir} commands= - -mkdir logs - pytest --junitxml xunit-results.xml --cov clamppolicyplugin --cov-report xml - coverage xml + pytest --junitxml xunit-results.xml --cov clamppolicyplugin --cov-report xml + coverage xml diff --git a/dcae-policy/dcaepolicyplugin/tasks.py b/dcae-policy/dcaepolicyplugin/tasks.py index 4e3b37a..2e62b15 100644 --- a/dcae-policy/dcaepolicyplugin/tasks.py +++ b/dcae-policy/dcaepolicyplugin/tasks.py @@ -1,5 +1,6 @@ # ================================================================================ # Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -194,11 +195,15 @@ def _policies_find(): policy_required = ctx.node.properties.get(POLICY_REQUIRED) try: - policy_filter = copy.deepcopy(dict( - (k, v) for (k, v) in dict(ctx.node.properties.get(POLICY_FILTER, {})).iteritems() - if v or isinstance(v, (int, float)) - )) - _fix_policy_filter(policy_filter) + policy_filter = ctx.node.properties.get(POLICY_FILTER) + if policy_filter: + policy_filter = { + k: copy.deepcopy(v) for k, v in policy_filter.items() + if v or isinstance(v, (int, float)) + } + _fix_policy_filter(policy_filter) + else: + policy_filter = {} if REQUEST_ID not in policy_filter: policy_filter[REQUEST_ID] = str(uuid.uuid4()) diff --git a/dcae-policy/requirements.txt b/dcae-policy/requirements.txt index 220b585..e9526b8 100644 --- a/dcae-policy/requirements.txt +++ b/dcae-policy/requirements.txt @@ -1 +1,3 @@ -requests>=2.11.0,<3.0.0 +requests>=2.11.0 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/dcae-policy/setup.py b/dcae-policy/setup.py index d7a6756..b95e303 100644 --- a/dcae-policy/setup.py +++ b/dcae-policy/setup.py @@ -1,5 +1,6 @@ # ================================================================================ # Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,7 +28,8 @@ setup( author='Alex Shatov', packages=['dcaepolicyplugin'], install_requires=[ - "requests>=2.11.0,<3.0.0" + 'requests>=2.11.0', + 'cloudify-common>=5.0.0', ], keywords='policy dcae controller cloudify plugin', classifiers=[ diff --git a/dcae-policy/tests/test_discovery.py b/dcae-policy/tests/test_discovery.py index 0c41fe3..07c652e 100644 --- a/dcae-policy/tests/test_discovery.py +++ b/dcae-policy/tests/test_discovery.py @@ -1,5 +1,6 @@ # ================================================================================ # Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -75,7 +76,7 @@ def monkeyed_discovery_get_kv(full_path, **kwargs): {tasks.DCAE_POLICY_PLUGIN: { tasks.PolicyHandler.SERVICE_NAME_POLICY_HANDLER: { "url": POLICY_HANDLER_FROM_KV}}} - )) + ).encode()).decode() return MonkeyedResponse(full_path, {}, [{"Value": value}]) return MonkeyedResponse(full_path) diff --git a/dcae-policy/tox-local.ini b/dcae-policy/tox-local.ini index 6bd1c58..d3ffbb0 100644 --- a/dcae-policy/tox-local.ini +++ b/dcae-policy/tox-local.ini @@ -1,15 +1,16 @@ # tox -c tox-local.ini | tee -a logs/test_dcaepolicyplugin.log 2>&1 [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= -rrequirements.txt - cloudify-plugins-common==3.4 pytest coverage pytest-cov -setenv = - PYTHONPATH={toxinidir} +whitelist_externals= + /bin/mkdir # recreate = True -commands=pytest -v --cov dcaepolicyplugin --cov-report html +commands= + mkdir -p logs + pytest -v --cov dcaepolicyplugin --cov-report html diff --git a/dcae-policy/tox.ini b/dcae-policy/tox.ini index 40caed5..42a4c39 100644 --- a/dcae-policy/tox.ini +++ b/dcae-policy/tox.ini @@ -1,17 +1,16 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= -rrequirements.txt - cloudify-plugins-common==3.4 pytest coverage pytest-cov -setenv = - PYTHONPATH={toxinidir} +whitelist_externals= + /bin/mkdir commands= - -mkdir logs - pytest --junitxml xunit-results.xml --cov dcaepolicyplugin --cov-report xml - coverage xml + mkdir -p logs + pytest --junitxml xunit-results.xml --cov dcaepolicyplugin --cov-report xml + coverage xml diff --git a/docker/dockerplugin/discovery.py b/docker/dockerplugin/discovery.py index 8361c13..563693c 100644 --- a/docker/dockerplugin/discovery.py +++ b/docker/dockerplugin/discovery.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -187,7 +188,7 @@ def is_healthy(consul_host, instance): def add_to_entry(conn, key, add_name, add_value): """ - Find 'key' in consul. + Find 'key' in consul. Treat its value as a JSON string representing a dict. Extend the dict by adding an entry with key 'add_name' and value 'add_value'. Turn the resulting extended dict into a JSON string. @@ -223,12 +224,9 @@ def add_to_entry(conn, key, add_name, add_value): def _find_matching_services(services, name_search, tags): """Find matching services given search criteria""" - def is_match(service): - srv_name, srv_tags = service - return name_search in srv_name and \ - all(map(lambda tag: tag in srv_tags, tags)) - - return [ srv[0] for srv in services.items() if is_match(srv) ] + tags = set(tags) + return [srv_name for srv_name in services + if name_search in srv_name and tags <= set(services[srv_name])] def search_services(conn, name_search, tags): """Search for services that match criteria diff --git a/docker/dockerplugin/tasks.py b/docker/dockerplugin/tasks.py index 03eba62..8a15319 100644 --- a/docker/dockerplugin/tasks.py +++ b/docker/dockerplugin/tasks.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -136,32 +137,29 @@ def _parse_streams(**kwargs): """Parse streams and setup for DMaaP plugin""" # The DMaaP plugin requires this plugin to set the runtime properties # keyed by the node name. - def setup_publishes(s): - kwargs[s["name"]] = s + for stream in kwargs["streams_publishes"]: + kwargs[stream["name"]] = stream - map(setup_publishes, kwargs["streams_publishes"]) - - def setup_subscribes(s): - if s["type"] == "data_router": - # If username and password has been provided then generate it. The - # DMaaP plugin doesn't generate for subscribers. The generation code - # and length of username password has been lifted from the DMaaP + # NOTE: That the delivery url is constructed and setup in the start operation + for stream in kwargs["streams_subscribes"]: + if stream["type"] == "data_router": + # If either username or password is missing then generate it. The + # DMaaP plugin doesn't generate them for subscribers. + # The code and length of username/password are lifted from the DMaaP # plugin. # Don't want to mutate the source - s = copy.deepcopy(s) - if not s.get("username", None): - s["username"] = utils.random_string(8) - if not s.get("password", None): - s["password"] = utils.random_string(10) + stream = copy.deepcopy(stream) + if not stream.get("username", None): + stream["username"] = utils.random_string(8) + if not stream.get("password", None): + stream["password"] = utils.random_string(10) - kwargs[s["name"]] = s - - # NOTE: That the delivery url is constructed and setup in the start operation - map(setup_subscribes, kwargs["streams_subscribes"]) + kwargs[stream["name"]] = stream return kwargs + def _setup_for_discovery_streams(**kwargs): """Setup for discovery of streams @@ -182,18 +180,17 @@ def _setup_for_discovery_streams(**kwargs): v = { "location": dr_sub["location"], "delivery_url": None, "username": dr_sub["username"], "password": dr_sub["password"], "subscriber_id": None } - return dis.add_to_entry(conn, dmaap_kv_key, dr_sub["name"], v) != None + return dis.add_to_entry(conn, dmaap_kv_key, dr_sub["name"], v) try: - if all(map(add_feed, dr_subs)): - return kwargs + for dr_sub in dr_subs: + if add_feed(dr_sub) is None: + raise NonRecoverableError( + "Failure updating feed streams in Consul") except Exception as e: raise NonRecoverableError(e) - # You should never get here - raise NonRecoverableError("Failure updating feed streams in Consul") - else: - return kwargs + return kwargs @merge_inputs_for_create diff --git a/docker/dockerplugin/utils.py b/docker/dockerplugin/utils.py index c45af68..6475aaa 100644 --- a/docker/dockerplugin/utils.py +++ b/docker/dockerplugin/utils.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,7 +35,7 @@ def update_dict(d, u): Update dict d with dict u """ - for k, v in u.iteritems(): + for k, v in u.items(): if isinstance(v, collections.Mapping): r = update_dict(d.get(k, {}), v) d[k] = r diff --git a/docker/requirements.txt b/docker/requirements.txt index 35db24e..36602c8 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,2 +1,5 @@ -onap-dcae-dockering==1.4.0 -onap-dcae-dcaepolicy-lib>=2.4.1,<3.0.0 +python-consul>=0.6.0 +onap-dcae-dockering>=1.4.0 +onap-dcae-dcaepolicy-lib>=2.4.1 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/docker/setup.py b/docker/setup.py index d301c49..8cb99b6 100644 --- a/docker/setup.py +++ b/docker/setup.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,9 +30,9 @@ setup( packages=['dockerplugin'], zip_safe=False, install_requires=[ - "python-consul>=0.6.0,<1.0.0", - "onap-dcae-dockering>=1.0.0,<2.0.0", - "uuid==1.30", - "onap-dcae-dcaepolicy-lib>=2.4.1,<3.0.0" + 'python-consul>=0.6.0', + 'onap-dcae-dockering>=1.4.0', + 'onap-dcae-dcaepolicy-lib>=2.4.1', + 'cloudify-common>=5.0.0', ] ) diff --git a/docker/tox.ini b/docker/tox.ini index 9e3cfb2..c38918a 100644 --- a/docker/tox.ini +++ b/docker/tox.ini @@ -1,16 +1,13 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= -rrequirements.txt - cloudify-plugins-common==3.4 pytest coverage pytest-cov -setenv= - PYTHONPATH={toxinidir} commands= - pytest --junitxml xunit-results.xml --cov dockerplugin --cov-report xml - coverage xml + pytest --junitxml xunit-results.xml --cov dockerplugin --cov-report xml + coverage xml diff --git a/k8s/configure/configure.py b/k8s/configure/configure.py index 9f7929e..959c215 100644 --- a/k8s/configure/configure.py +++ b/k8s/configure/configure.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2018-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -75,13 +76,16 @@ def configure(config_path=_CONFIG_PATH, key = _CONSUL_KEY): """ from cloudify.exceptions import NonRecoverableError - import ConfigParser + try: + import configparser + except ImportError: + import ConfigParser as configparser from k8splugin import discovery config = _set_defaults() try: # Get Consul address from a config file - c = ConfigParser.ConfigParser() + c = configparser.ConfigParser() c.read(config_path) config["consul_host"] = c.get('consul','address') diff --git a/k8s/k8sclient/k8sclient.py b/k8s/k8sclient/k8sclient.py index 9a01536..323a208 100644 --- a/k8s/k8sclient/k8sclient.py +++ b/k8s/k8sclient/k8sclient.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -136,7 +137,7 @@ def _create_resources(resources=None): def _create_container_object(name, image, always_pull, env={}, container_ports=[], volume_mounts = [], resources = None, readiness = None, liveness = None): # Set up environment variables # Copy any passed in environment variables - env_vars = [client.V1EnvVar(name=k, value=env[k]) for k in env.keys()] + env_vars = [client.V1EnvVar(name=k, value=env[k]) for k in env] # Add POD_IP with the IP address of the pod running the container pod_ip = client.V1EnvVarSource(field_ref = client.V1ObjectFieldSelector(field_path="status.podIP")) env_vars.append(client.V1EnvVar(name="POD_IP",value_from=pod_ip)) @@ -517,7 +518,7 @@ def deploy(namespace, component_name, image, replicas, always_pull, k8sconfig, r if port_map: service_ports = [] # Ports exposed internally on the k8s network exposed_ports = [] # Ports to be mapped to ports on the k8s nodes via NodePort - for (cport, proto), hport in port_map.iteritems(): + for (cport, proto), hport in port_map.items(): service_ports.append(client.V1ServicePort(port=int(cport),protocol=proto,name="port-{0}-{1}".format(proto[0].lower(), cport))) if int(hport) != 0: exposed_ports.append(client.V1ServicePort(port=int(cport),protocol=proto,node_port=int(hport),name="xport-{0}-{1}".format(proto[0].lower(),cport))) @@ -660,8 +661,6 @@ def execute_command_in_deployment(deployment_description, command): field_selector = "status.phase=Running" ).items] - def do_execute(pod_name): - return _execute_command_in_pod(location, namespace, pod_name, command) - # Execute command in the running pods - return map(do_execute, pod_names) + return [_execute_command_in_pod(location, namespace, pod_name, command) + for pod_name in pod_names] diff --git a/k8s/k8splugin/discovery.py b/k8s/k8splugin/discovery.py index 56f8260..76c160a 100644 --- a/k8s/k8splugin/discovery.py +++ b/k8s/k8splugin/discovery.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -233,12 +234,10 @@ def add_to_entry(conn, key, add_name, add_value): def _find_matching_services(services, name_search, tags): """Find matching services given search criteria""" - def is_match(service): - srv_name, srv_tags = service - return name_search in srv_name and \ - all(map(lambda tag: tag in srv_tags, tags)) + tags = set(tags) + return [srv_name for srv_name in services + if name_search in srv_name and tags <= set(services[srv_name])] - return [ srv[0] for srv in services.items() if is_match(srv) ] def search_services(conn, name_search, tags): """Search for services that match criteria diff --git a/k8s/k8splugin/tasks.py b/k8s/k8splugin/tasks.py index 108cf31..956fff2 100644 --- a/k8s/k8splugin/tasks.py +++ b/k8s/k8splugin/tasks.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,9 +21,6 @@ # Lifecycle interface calls for containerized components -# Needed by Cloudify Manager to load google.auth for the Kubernetes python client -import cloudify_importer - import time, copy import json from cloudify import ctx @@ -138,16 +136,14 @@ def _parse_streams(**kwargs): """Parse streams and setup for DMaaP plugin""" # The DMaaP plugin requires this plugin to set the runtime properties # keyed by the node name. - def setup_publishes(s): - kwargs[s["name"]] = s - - map(setup_publishes, kwargs["streams_publishes"]) + for stream in kwargs["streams_publishes"]: + kwargs[stream["name"]] = stream - def setup_subscribes(s): - if s["type"] == "data_router": + for stream in kwargs["streams_subscribes"]: + if stream["type"] == "data_router": # Don't want to mutate the source - s = copy.deepcopy(s) + stream = copy.deepcopy(stream) # Set up the delivery URL # Using service_component_name as the host name in the subscriber URL @@ -156,29 +152,27 @@ def _parse_streams(**kwargs): # more remote ("edge") locations depends on how networking and DNS is set # up in a multi-cluster deployment service_component_name = kwargs["name"] - ports,_ = k8sclient.parse_ports(kwargs["ports"]) - (dport, _) = ports[0] + ports, _ = k8sclient.parse_ports(kwargs["ports"]) + dport, _ = ports[0] subscriber_host = "{host}:{port}".format(host=service_component_name, port=dport) - scheme = s["scheme"] if "scheme" in s else DEFAULT_SCHEME - if "route" not in s: + scheme = stream.get("scheme", DEFAULT_SCHEME) + if "route" not in stream: raise NonRecoverableError("'route' key missing from data router subscriber") - path = s["route"] - s["delivery_url"] = "{scheme}://{host}/{path}".format( + path = stream["route"] + stream["delivery_url"] = "{scheme}://{host}/{path}".format( scheme=scheme, host=subscriber_host, path=path) # If username and password has not been provided then generate it. The # DMaaP plugin doesn't generate for subscribers. The generation code # and length of username password has been lifted from the DMaaP # plugin. - if not s.get("username", None): - s["username"] = utils.random_string(8) - if not s.get("password", None): - s["password"] = utils.random_string(10) - - kwargs[s["name"]] = s + if not stream.get("username", None): + stream["username"] = utils.random_string(8) + if not stream.get("password", None): + stream["password"] = utils.random_string(10) - map(setup_subscribes, kwargs["streams_subscribes"]) + kwargs[stream["name"]] = stream return kwargs diff --git a/k8s/k8splugin/utils.py b/k8s/k8splugin/utils.py index c45af68..6475aaa 100644 --- a/k8s/k8splugin/utils.py +++ b/k8s/k8splugin/utils.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,7 +35,7 @@ def update_dict(d, u): Update dict d with dict u """ - for k, v in u.iteritems(): + for k, v in u.items(): if isinstance(v, collections.Mapping): r = update_dict(d.get(k, {}), v) d[k] = r diff --git a/k8s/requirements.txt b/k8s/requirements.txt index b2c0986..2a94ae6 100644 --- a/k8s/requirements.txt +++ b/k8s/requirements.txt @@ -1,5 +1,5 @@ -python-consul>=0.6.0,<1.0.0 -uuid==1.30 -onap-dcae-dcaepolicy-lib>=2.4.1,<3.0.0 -kubernetes==9.0.0 -cloudify-plugins-common==3.4 +python-consul>=0.6.0 +onap-dcae-dcaepolicy-lib>=2.4.1 +kubernetes>=9.0.0 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/k8s/setup.py b/k8s/setup.py index b0e43f6..386dde8 100644 --- a/k8s/setup.py +++ b/k8s/setup.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,11 +29,9 @@ setup( packages=['k8splugin','k8sclient','msb','configure'], zip_safe=False, install_requires=[ - "python-consul>=0.6.0,<1.0.0", - "uuid==1.30", - "onap-dcae-dcaepolicy-lib>=2.4.1,<3.0.0", - "cloudify-plugins-common==3.4", - "cloudify-python-importer==0.1.0", - "kubernetes==9.0.0" + 'python-consul>=0.6.0', + 'onap-dcae-dcaepolicy-lib>=2.4.1', + 'kubernetes>=9.0.0', + 'cloudify-common>=5.0.0', ] ) diff --git a/k8s/tests/test_k8sclient.py b/k8s/tests/test_k8sclient.py index 079748c..fcc6b95 100644 --- a/k8s/tests/test_k8sclient.py +++ b/k8s/tests/test_k8sclient.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2018-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -34,9 +35,9 @@ def test_parse_interval(): (354123, 354123), ("354123", 354123), ("354123s", 354123), - (1234567890123456789012345678901234567890L,1234567890123456789012345678901234567890L), - ("1234567890123456789012345678901234567890",1234567890123456789012345678901234567890L), - ("1234567890123456789012345678901234567890s",1234567890123456789012345678901234567890L), + (1234567890123456789012345678901234567890,1234567890123456789012345678901234567890), + ("1234567890123456789012345678901234567890",1234567890123456789012345678901234567890), + ("1234567890123456789012345678901234567890s",1234567890123456789012345678901234567890), ("05s", 5), ("00000000000000000000000000000000005m", 5 * 60) ] diff --git a/k8s/tox.ini b/k8s/tox.ini index b6b2b54..3bea42d 100644 --- a/k8s/tox.ini +++ b/k8s/tox.ini @@ -1,6 +1,6 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= @@ -8,8 +8,6 @@ deps= pytest coverage pytest-cov -setenv= - PYTHONPATH={toxinidir} commands= - pytest --junitxml xunit-results.xml --cov configure --cov k8sclient --cov k8splugin --cov msb --cov-report xml - coverage xml + pytest --junitxml xunit-results.xml --cov configure --cov k8sclient --cov k8splugin --cov msb --cov-report xml + coverage xml diff --git a/relationships/relationshipplugin/discovery.py b/relationships/relationshipplugin/discovery.py index bd0e369..1b87a65 100644 --- a/relationships/relationshipplugin/discovery.py +++ b/relationships/relationshipplugin/discovery.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +19,10 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. -from urlparse import urlparse +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse import json import consul diff --git a/relationships/requirements.txt b/relationships/requirements.txt index 5d5d52e..2a92bd4 100644 --- a/relationships/requirements.txt +++ b/relationships/requirements.txt @@ -1 +1,3 @@ -python-consul==0.6.1 +python-consul>=0.6.1 +cloudify-common>=5.0.0; python_version<"3" +cloudify-common @ git+https://github.com/cloudify-cosmo/cloudify-common@cy-1374-python3#egg=cloudify-common==5.0.0; python_version>="3" diff --git a/relationships/setup.py b/relationships/setup.py index e576d3c..7facd65 100644 --- a/relationships/setup.py +++ b/relationships/setup.py @@ -2,6 +2,7 @@ # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019 Pantheon.tech. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,6 +30,7 @@ setup( packages=['relationshipplugin'], zip_safe=False, install_requires=[ - "python-consul>=0.6.0" + 'python-consul>=0.6.1', + 'cloudify-common>=5.0.0', ] ) diff --git a/relationships/tox.ini b/relationships/tox.ini index a7157be..499accb 100644 --- a/relationships/tox.ini +++ b/relationships/tox.ini @@ -1,15 +1,13 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 +envlist = py27,py36 [testenv] deps= - cloudify-plugins-common==3.4 + -rrequirements.txt pytest coverage pytest-cov -setenv= - PYTHONPATH={toxinidir} commands= - pytest --junitxml xunit-results.xml --cov relationshipplugin --cov-report xml - coverage xml + pytest --junitxml xunit-results.xml --cov relationshipplugin --cov-report xml + coverage xml -- cgit 1.2.3-korg