diff options
Diffstat (limited to 'mod')
36 files changed, 722 insertions, 383 deletions
diff --git a/mod/distributorapi/Changelog.md b/mod/distributorapi/Changelog.md index f0fb015..fc1ec2e 100644 --- a/mod/distributorapi/Changelog.md +++ b/mod/distributorapi/Changelog.md @@ -4,7 +4,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.1.1] - 2022/09/23 + * DCAEGEN2-3170 - CodeCoverage improvement for dcaegen2-platform-mod-distributorapi + * fix value of MarkupSafe to 2.0.1 because jango+jinja uses a slightly older API + * run black on mod/distributorapi + + * DCAEGEN2-3271 - Switch to 9.1.0 base image which used python 3.9 + +## [1.1.0] - 2020/08/19 + * DCAEGEN2-2292 - ONAP must complete update of the Python language (from 2.7 -> 3.8) + ## [1.0.1] - 2020/3/26 * Run as non-root + ## [1.0.0] - 2019/11/11 * Distributor API - initial version diff --git a/mod/distributorapi/Dockerfile b/mod/distributorapi/Dockerfile index 84ee6b5..29fbfb3 100644 --- a/mod/distributorapi/Dockerfile +++ b/mod/distributorapi/Dockerfile @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019,2022 AT&T Intellectual Property. All rights reserved. # Copyright 2020 Deutsche Telekom. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============LICENSE_END========================================================= -FROM nexus3.onap.org:10001/onap/integration-python:latest +FROM nexus3.onap.org:10001/onap/integration-python:9.1.0 COPY --chown=onap:onap . /app RUN pip install . diff --git a/mod/distributorapi/distributor/config.py b/mod/distributorapi/distributor/config.py index d83eee1..2195220 100644 --- a/mod/distributorapi/distributor/config.py +++ b/mod/distributorapi/distributor/config.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,11 +28,10 @@ def _grab_env(name, default=None): except KeyError: raise errors.DistributorAPIConfigError("Required environment variable missing: {0}".format(name)) + def init(): global nifi_registry_url - nifi_registry_url = _grab_env("NIFI_REGISTRY_URL" - , default="http://nifi-registry:18080/nifi-registry-api") + nifi_registry_url = _grab_env("NIFI_REGISTRY_URL", default="http://nifi-registry:18080/nifi-registry-api") global onboarding_api_url - onboarding_api_url = _grab_env("ONBOARDING_API_URL" - , default="http://onboarding-api:8080/onboarding") + onboarding_api_url = _grab_env("ONBOARDING_API_URL", default="http://onboarding-api:8080/onboarding") diff --git a/mod/distributorapi/distributor/data_access.py b/mod/distributorapi/distributor/data_access.py index e1a45e3..5b544d8 100644 --- a/mod/distributorapi/distributor/data_access.py +++ b/mod/distributorapi/distributor/data_access.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,9 +29,10 @@ def get_distribution_targets(): def get_distribution_target(ds_id): global _cache - result = [ i for i in _cache if i["dt_id"] == ds_id ] + result = [i for i in _cache if i["dt_id"] == ds_id] return result[0] if result else {} + def transform_request(req): """Transform request to object to store @@ -44,6 +45,7 @@ def transform_request(req): req["processGroups"] = [] return req + def add_distribution_target(dt): global _cache _cache.append(dt) @@ -58,6 +60,7 @@ def merge_request(dt, req): dt["modified"] = datetime.utcnow().isoformat() return dt + def update_distribution_target(updated_dt): dt_id = updated_dt["dt_id"] global _cache @@ -85,5 +88,3 @@ def add_process_group(ds_id, process_group): dt["processGroups"].append(process_group) return process_group return None - - diff --git a/mod/distributorapi/distributor/errors.py b/mod/distributorapi/distributor/errors.py index e28b5f5..04779b0 100644 --- a/mod/distributorapi/distributor/errors.py +++ b/mod/distributorapi/distributor/errors.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,11 +15,14 @@ # ============LICENSE_END========================================================= """Errors""" + class DistributorAPIError(RuntimeError): pass + class DistributorAPIConfigError(DistributorAPIError): pass + class DistributorAPIResourceNotFound(DistributorAPIError): pass diff --git a/mod/distributorapi/distributor/http.py b/mod/distributorapi/distributor/http.py index f1aa2fd..e367b19 100644 --- a/mod/distributorapi/distributor/http.py +++ b/mod/distributorapi/distributor/http.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -36,64 +36,82 @@ _app = Flask(__name__) CORS(_app) # Try to bundle as many errors together # https://flask-restplus.readthedocs.io/en/stable/parsing.html#error-handling -_app.config['BUNDLE_ERRORS'] = True -_api = Api(_app, version=__version__, title="Distributor HTTP API", - description="HTTP API to manage distribution targets for DCAE design. Distribution targets are DCAE runtime environments that have been registered and are enabled to accept flow design changes that are to be orchestrated in that environment", - contact="", default_mediatype="application/json" - , prefix="/distributor", doc="/distributor", default="distributor" - ) +_app.config["BUNDLE_ERRORS"] = True +_api = Api( + _app, + version=__version__, + title="Distributor HTTP API", + description="HTTP API to manage distribution targets for DCAE design. Distribution targets are DCAE runtime environments that have been registered and are enabled to accept flow design changes that are to be orchestrated in that environment", + contact="", + default_mediatype="application/json", + prefix="/distributor", + doc="/distributor", + default="distributor", +) # REVIEW: Do I need a namespace? ns = _api -model_pg = _api.model("ProcessGroup", { - "id": fields.String(required=True, description="Id for this process group" - , attribute="processGroupId") - , "version": fields.Integer(required=True - , description="Version of the process group") - , "processed": fields.DateTime(required=True - , description="When this process group was processed by this API") - , "runtimeResponse": fields.String(required=True - , description="Full response from the runtime API") - }) - -model_dt = _api.model("DistributionTarget", { - "selfUrl": fields.Url("resource_distribution_target", absolute=True) - , "id": fields.String(required=True, description="Id for this distribution target" - , attribute="dt_id") - , "name": fields.String(required=True, description="Name for this distribution target" - , attribute="name") - , "runtimeApiUrl": fields.String(required=True - , description="Url to the runtime API for this distribution target" - , attribute="runtimeApiUrl") - , "description": fields.String(required=False - , description="Description for this distribution target" - , attribute="description") - , "nextDistributionTargetId": fields.String(required=False - , description="Id to the next distribution target. Distribution targets can be linked together and have a progression order. Specifying the id of the next distribution target defines the next element int the order." - , attribute="nextDistributionTargetId") - , "created": fields.String(required=True - , description="When this distribution target was created in UTC" - , attribute="created") - , "modified": fields.String(required=True - , description="When this distribution target was last modified in UTC" - , attribute="modified") - , "processGroups": fields.List(fields.Nested(model_pg)) - }) - -model_dts = _api.model("DistributionTargets", { - "distributionTargets": fields.List(fields.Nested(model_dt)) - }) +model_pg = _api.model( + "ProcessGroup", + { + "id": fields.String(required=True, description="Id for this process group", attribute="processGroupId"), + "version": fields.Integer(required=True, description="Version of the process group"), + "processed": fields.DateTime(required=True, description="When this process group was processed by this API"), + "runtimeResponse": fields.String(required=True, description="Full response from the runtime API"), + }, +) + +model_dt = _api.model( + "DistributionTarget", + { + "selfUrl": fields.Url("resource_distribution_target", absolute=True), + "id": fields.String(required=True, description="Id for this distribution target", attribute="dt_id"), + "name": fields.String(required=True, description="Name for this distribution target", attribute="name"), + "runtimeApiUrl": fields.String( + required=True, description="Url to the runtime API for this distribution target", attribute="runtimeApiUrl" + ), + "description": fields.String( + required=False, description="Description for this distribution target", attribute="description" + ), + "nextDistributionTargetId": fields.String( + required=False, + description="Id to the next distribution target. Distribution targets can be linked together and have a progression order. Specifying the id of the next distribution target defines the next element int the order.", + attribute="nextDistributionTargetId", + ), + "created": fields.String( + required=True, description="When this distribution target was created in UTC", attribute="created" + ), + "modified": fields.String( + required=True, description="When this distribution target was last modified in UTC", attribute="modified" + ), + "processGroups": fields.List(fields.Nested(model_pg)), + }, +) + +model_dts = _api.model("DistributionTargets", {"distributionTargets": fields.List(fields.Nested(model_dt))}) parser_dt_req = ns.parser() -parser_dt_req.add_argument("name", required=True, trim=True, - location="json", help="Name for this new distribution target") -parser_dt_req.add_argument("runtimeApiUrl", required=True, trim=True, - location="json", help="Url to the runtime API for this distribution target") -parser_dt_req.add_argument("description", required=False, trim=True, - location="json", help="Description for this distribution target") -parser_dt_req.add_argument("nextDistributionTargetId", required=False, trim=True, - location="json", help="Id of the next distribution target. Distribution targets can be linked together and have a progression order. Specifying the id of the next distribution target defines the next element int the order.") +parser_dt_req.add_argument( + "name", required=True, trim=True, location="json", help="Name for this new distribution target" +) +parser_dt_req.add_argument( + "runtimeApiUrl", + required=True, + trim=True, + location="json", + help="Url to the runtime API for this distribution target", +) +parser_dt_req.add_argument( + "description", required=False, trim=True, location="json", help="Description for this distribution target" +) +parser_dt_req.add_argument( + "nextDistributionTargetId", + required=False, + trim=True, + location="json", + help="Id of the next distribution target. Distribution targets can be linked together and have a progression order. Specifying the id of the next distribution target defines the next element int the order.", +) @ns.route("/distribution-targets", endpoint="resource_distribution_targets") @@ -101,7 +119,7 @@ class DistributionTargets(Resource): @ns.doc("get_distribution_targets", description="List distribution targets") @ns.marshal_with(model_dts) def get(self): - return { "distributionTargets": da.get_distribution_targets() }, 200 + return {"distributionTargets": da.get_distribution_targets()}, 200 @ns.doc("post_distribution_targets", description="Create a new distribution target") @ns.expect(parser_dt_req) @@ -112,11 +130,12 @@ class DistributionTargets(Resource): resp = da.add_distribution_target(req) return resp, 200 + @ns.route("/distribution-targets/<string:dt_id>", endpoint="resource_distribution_target") class DistributionTarget(Resource): @ns.doc("get_distribution_target", description="Get a distribution target instance") - @ns.response(404, 'Distribution target not found') - @ns.response(500, 'Internal Server Error') + @ns.response(404, "Distribution target not found") + @ns.response(500, "Internal Server Error") @ns.marshal_with(model_dt) def get(self, dt_id): result = da.get_distribution_target(dt_id) @@ -127,8 +146,8 @@ class DistributionTarget(Resource): frp.abort(code=404, message="Unknown distribution target") @ns.doc("put_distribution_target", description="Update an existing distribution target") - @ns.response(404, 'Distribution target not found') - @ns.response(500, 'Internal Server Error') + @ns.response(404, "Distribution target not found") + @ns.response(500, "Internal Server Error") @ns.expect(parser_dt_req) @ns.marshal_with(model_dt) def put(self, dt_id): @@ -145,8 +164,8 @@ class DistributionTarget(Resource): else: frp.abort(code=500, message="Problem with storing the update") - @ns.response(404, 'Distribution target not found') - @ns.response(500, 'Internal Server Error') + @ns.response(404, "Distribution target not found") + @ns.response(500, "Internal Server Error") @ns.doc("delete_distribution_target", description="Delete an existing distribution target") def delete(self, dt_id): if da.delete_distribution_target(dt_id): @@ -156,15 +175,16 @@ class DistributionTarget(Resource): parser_post_process_group = ns.parser() -parser_post_process_group.add_argument("processGroupId", required=True, - trim=True, location="json", help="Process group ID that exists in Nifi") +parser_post_process_group.add_argument( + "processGroupId", required=True, trim=True, location="json", help="Process group ID that exists in Nifi" +) + @ns.route("/distribution-targets/<string:dt_id>/process-groups", endpoint="resource_target_process_groups") class DTargetProcessGroups(Resource): - - @ns.response(404, 'Distribution target not found') - @ns.response(501, 'Feature is not supported right now') - @ns.response(500, 'Internal Server Error') + @ns.response(404, "Distribution target not found") + @ns.response(501, "Feature is not supported right now") + @ns.response(500, "Internal Server Error") @ns.expect(parser_post_process_group) def post(self, dt_id): # TODO: Need bucket ID but for now will simply scan through all buckets @@ -198,7 +218,7 @@ class DTargetProcessGroups(Resource): # Make sure graph is setup in runtime api if runc.ensure_graph(runtime_url, pg_id, pg_name) == False: - frp.abort(code=501 , message="Runtime API: Graph could not be created") + frp.abort(code=501, message="Runtime API: Graph could not be created") # Graph diffing using Nifi registry @@ -245,6 +265,7 @@ def start_http_server(): def is_debug(): import os + if os.environ.get("DISTRIBUTOR_DEBUG", "1") == "1": return True else: diff --git a/mod/distributorapi/distributor/onboarding_client.py b/mod/distributorapi/distributor/onboarding_client.py index d7b0780..9665561 100644 --- a/mod/distributorapi/distributor/onboarding_client.py +++ b/mod/distributorapi/distributor/onboarding_client.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -30,6 +30,4 @@ def get_component(onboarding_url, name, version): def get_components_indexed(onboarding_url, list_name_version): - return dict([ - ((c[0], c[1]), get_component(onboarding_url, c[0], c[1])) - for c in list_name_version]) + return dict([((c[0], c[1]), get_component(onboarding_url, c[0], c[1])) for c in list_name_version]) diff --git a/mod/distributorapi/distributor/registry_client.py b/mod/distributorapi/distributor/registry_client.py index 5d437e7..372099a 100644 --- a/mod/distributorapi/distributor/registry_client.py +++ b/mod/distributorapi/distributor/registry_client.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ def _add_url_from_link(registry_url, obj): for k, v in obj.items(): if k == "link": - result["selfUrl"] =_urljoin(registry_url, v["href"]) + result["selfUrl"] = _urljoin(registry_url, v["href"]) result[k] = v elif type(v) == dict: result[k] = _add_url_from_link(registry_url, v) @@ -63,13 +63,14 @@ def get_flow_versions(flow_url): """Returns list of versions from greatest to least for a given flow""" versions_url = _urljoin(flow_url, "versions") # List of versions will be greatest to least - return list(reversed(sorted( - [v["version"] for v in _get_json(versions_url)]))) + return list(reversed(sorted([v["version"] for v in _get_json(versions_url)]))) + def get_flow_diff(registry_url, flow_url, version_one, version_two): diff_url = _urljoin(flow_url, "diff", str(version_one), str(version_two)) return _get_json(diff_url) + def get_flow_diff_latest(registry_url, flow_url): versions = get_flow_versions(flow_url) @@ -80,12 +81,13 @@ def get_flow_diff_latest(registry_url, flow_url): return None else: # Example in gitlab wiki shows that lower version is first - return _add_url_from_link(registry_url - , get_flow_diff(registry_url, flow_url, versions[1], versions[0])) + return _add_url_from_link(registry_url, get_flow_diff(registry_url, flow_url, versions[1], versions[0])) + def get_flow_version(registry_url, flow_url, version): version_url = _urljoin(flow_url, "versions", str(version)) return _add_url_from_link(registry_url, _get_json(version_url)) + def get_flow_version_latest(registry_url, flow_url): return get_flow_version(registry_url, flow_url, "latest") diff --git a/mod/distributorapi/distributor/runtime_client.py b/mod/distributorapi/distributor/runtime_client.py index 7cd06ae..2fc2ccd 100644 --- a/mod/distributorapi/distributor/runtime_client.py +++ b/mod/distributorapi/distributor/runtime_client.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -29,8 +29,7 @@ def get_graph(runtime_url, graph_id): def create_graph(runtime_url, graph_id, graph_name): url = urljoin(runtime_url, "api/graph/main") - resp = reqs.post(url, json={"name": graph_name, "id": graph_id - , "description": "", "main": True}) + resp = reqs.post(url, json={"name": graph_name, "id": graph_id, "description": "", "main": True}) try: resp.raise_for_status() @@ -60,6 +59,7 @@ def post_graph(runtime_url, graph_id, actions): except Exception as e: with open("runtime-request-failed.json", "w+") as f: import json + json.dump(graph_request, f) raise errors.DistributorAPIError(e) diff --git a/mod/distributorapi/distributor/transform.py b/mod/distributorapi/distributor/transform.py index 9654249..5ca1277 100644 --- a/mod/distributorapi/distributor/transform.py +++ b/mod/distributorapi/distributor/transform.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ def extract_components_from_flow(flow): and create a list of tuples where each tuple is (component name, component version)""" extract = lambda p: (p["bundle"]["artifact"], p["bundle"]["version"]) - return [ extract(p) for p in flow["flowContents"]["processors"] ] + return [extract(p) for p in flow["flowContents"]["processors"]] def get_component(flow, components, processor_id): @@ -32,8 +32,7 @@ def get_component(flow, components, processor_id): bundle = p["bundle"] return components.get((bundle["artifact"], bundle["version"]), None) - cs = [get_component(p) for p in flow["flowContents"]["processors"] \ - if p["identifier"] == processor_id] + cs = [get_component(p) for p in flow["flowContents"]["processors"] if p["identifier"] == processor_id] return cs[0] if cs else None @@ -80,10 +79,10 @@ def make_fbp_from_flow(flow, components: "dict of (name, version) to components" if rels_pubs: _, _, _, transport_type, config_key = rels_pubs[0].split(":") - src = { "node": comp["id"], "port": config_key } + src = {"node": comp["id"], "port": config_key} else: # REVIEW: This should be an error? - src = { "node": comp["id"], "port": None } + src = {"node": comp["id"], "port": None} else: src = {} @@ -100,38 +99,43 @@ def make_fbp_from_flow(flow, components: "dict of (name, version) to components" if rels_subs: _, _, _, transport_type, config_key = rels_subs[0].split(":") - tgt = { "node": comp["id"], "port": config_key } + tgt = {"node": comp["id"], "port": config_key} else: # REVIEW: This should be an error? - tgt = { "node": comp["id"], "port": None } + tgt = {"node": comp["id"], "port": None} else: tgt = {} - return { "command": "addedge" - , "payload": { - "src": src - , "tgt": tgt - , "metadata": { - "name": conn["name"] - # TODO: Question these hardcoded attributes - , "data_type": "json" - , "dmaap_type": "MR" - } - } - } + return { + "command": "addedge", + "payload": { + "src": src, + "tgt": tgt, + "metadata": { + "name": conn["name"] + # TODO: Question these hardcoded attributes + , + "data_type": "json", + "dmaap_type": "MR", + }, + }, + } def parse_processor(p): c = components[(p["bundle"]["artifact"], p["bundle"]["version"])] - return { "command": "addnode" - # TODO: spec is required to be a json string but runtime api - # changing this soon hopefully - , "payload": { "component_spec": json.dumps(c["spec"]) - , "component_id": c["id"] - , "name": c["name"] - , "processor": p } - } - - ps = [ parse_processor(p) for p in flow["flowContents"]["processors"] ] - cs = [ parse_connection(c) for c in flow["flowContents"]["connections"] ] - return ps+cs + return { + "command": "addnode" + # TODO: spec is required to be a json string but runtime api + # changing this soon hopefully + , + "payload": { + "component_spec": json.dumps(c["spec"]), + "component_id": c["id"], + "name": c["name"], + "processor": p, + }, + } + ps = [parse_processor(p) for p in flow["flowContents"]["processors"]] + cs = [parse_connection(c) for c in flow["flowContents"]["connections"]] + return ps + cs diff --git a/mod/distributorapi/distributor/utils.py b/mod/distributorapi/distributor/utils.py index 7457d5a..c8449bf 100644 --- a/mod/distributorapi/distributor/utils.py +++ b/mod/distributorapi/distributor/utils.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ def urljoin(base, *trailing, **query_params): url = "/".join(full) if query_params: - qp = ["{0}={1}".format(quote(k), quote(str(v))) for k,v in query_params.items()] + qp = ["{0}={1}".format(quote(k), quote(str(v))) for k, v in query_params.items()] qp = "&".join(qp) return "?".join([url, qp]) else: diff --git a/mod/distributorapi/distributor/version.py b/mod/distributorapi/distributor/version.py index 9da6f0f..bef5bf5 100644 --- a/mod/distributorapi/distributor/version.py +++ b/mod/distributorapi/distributor/version.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============LICENSE_END========================================================= -__version__ = "1.0.1" +__version__ = "1.1.1" diff --git a/mod/distributorapi/pom.xml b/mod/distributorapi/pom.xml index 75ed7a7..423921c 100644 --- a/mod/distributorapi/pom.xml +++ b/mod/distributorapi/pom.xml @@ -3,7 +3,7 @@ ============LICENSE_START======================================================= org.onap.dcae ================================================================================ -Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. Copyright 2020 Deutsche Telekom. All rights reserved. ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,19 +20,19 @@ limitations under the License. ============LICENSE_END========================================================= --> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <groupId>org.onap.dcaegen2.platform.mod</groupId> + <modelVersion>4.0.0</modelVersion> + <groupId>org.onap.dcaegen2.platform.mod</groupId> <artifactId>dcaegen2-platform-mod-distributorapi</artifactId> <!-- NOTE: Must keep this version synchronized with the version in distributor/version.py file --> - <version>1.1.0-SNAPSHOT</version> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <sonar.sources>.</sonar.sources> - <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> - <sonar.python.coverage.reportPaths>coverage.xml</sonar.python.coverage.reportPaths> - <sonar.language>py</sonar.language> - <sonar.pluginname>python</sonar.pluginname> - <sonar.inclusions>**/*.py</sonar.inclusions> - <sonar.exclusions>**/tests/**,**/setup.py</sonar.exclusions> - </properties> + <version>1.1.1-SNAPSHOT</version> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <sonar.sources>.</sonar.sources> + <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> + <sonar.python.coverage.reportPaths>coverage.xml</sonar.python.coverage.reportPaths> + <sonar.language>py</sonar.language> + <sonar.pluginname>python</sonar.pluginname> + <sonar.inclusions>**/*.py</sonar.inclusions> + <sonar.exclusions>**/tests/**,**/setup.py</sonar.exclusions> + </properties> </project> diff --git a/mod/distributorapi/setup.py b/mod/distributorapi/setup.py index 4a09471..201aedb 100644 --- a/mod/distributorapi/setup.py +++ b/mod/distributorapi/setup.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,26 +18,21 @@ from setuptools import setup, find_packages # extract __version__ from version file. importing distributor will lead to install failures setup_dir = os.path.dirname(__file__) -with open(os.path.join(setup_dir, 'distributor', 'version.py')) as file: +with open(os.path.join(setup_dir, "distributor", "version.py")) as file: globals_dict = dict() exec(file.read(), globals_dict) - __version__ = globals_dict['__version__'] + __version__ = globals_dict["__version__"] setup( - name = "distributor-api", - version = __version__, - packages = find_packages(), - author = "Michael Hwang", - description = ("API that manages distribution targets"), - entry_points=""" + name="distributor-api", + version=__version__, + packages=find_packages(), + author="Michael Hwang", + description=("API that manages distribution targets"), + entry_points=""" [console_scripts] start-distributor-api=distributor.http:start_http_server """, - install_requires=[ - "Werkzeug==0.16.1", - "flask-restplus" - , "Flask-Cors" - , "requests" - ], - zip_safe = False - ) + install_requires=["Werkzeug==0.16.1", "flask-restplus", "Flask-Cors", "requests", "MarkupSafe==2.0.1"], + zip_safe=False, +) diff --git a/mod/distributorapi/tests/test_api.py b/mod/distributorapi/tests/test_api.py index d7f543e..e943c04 100644 --- a/mod/distributorapi/tests/test_api.py +++ b/mod/distributorapi/tests/test_api.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2020 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2020-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,103 +14,282 @@ # limitations under the License. # ============LICENSE_END========================================================= -from distributor.http import _app as app -from distributor import config +""" +Tests that require a mock requests module, plus a few more +that didn't fit cleanly elsewhere. +""" + +import copy +import os +import re import pytest import requests +from distributor.http import _app as app +from distributor import config +from distributor import onboarding_client +from distributor import utils +from distributor import errors +from distributor import data_access +from distributor import transform + class _resp(object): - def __init__(self, code, json = None): + def __init__(self, code, json=None): self.status_code = code if json is not None: self._json = json - + def json(self): return self._json def raise_for_status(self): if self.status_code < 200 or self.status_code >= 300: - raise Exception('Error response {}'.format(self.status_code)) + raise Exception("Error response {}".format(self.status_code)) + class _req(object): + # in the test code, you can set + # _req.SHOWMATCHES = True + # and the match results will be displayed + SHOWMATCHES = False + def __init__(self, op, url, resp): self.op = op - self.url = url; + self.url = url self.resp = resp def check(self, op, url): - if op != self.op or url != self.url: - return None - return self.resp + if _req.SHOWMATCHES: + print(f"_req.check(op={op} vs {self.op}, url={url} vs {self.url})") + return self.resp if op == self.op and url == self.url else None + def _match(answers, op, url): for choice in answers: ret = choice.check(op, url) if ret is not None: return ret - message = 'Unexpected request {} {}'.format(op, url) + message = "Unexpected request {} {}".format(op, url) print(message) raise Exception(message) + @pytest.fixture def mockrequests(monkeypatch): answers = [] - def get(url, headers = None): - return _match(answers, 'GET', url) - - def post(url, json, headers = None): - return _match(answers, 'POST', url) - - def put(url, json, headers = None): - return _match(answers, 'PUT', url) - - def delete(url, headers = None): - return _match(answers, 'DELETE', url) - - monkeypatch.setattr(requests, 'get', get) - monkeypatch.setattr(requests, 'post', post) - monkeypatch.setattr(requests, 'put', put) - monkeypatch.setattr(requests, 'delete', delete) + + def get(url, headers=None): + return _match(answers, "GET", url) + + def post(url, json, headers=None): + return _match(answers, "POST", url) + + def put(url, json, headers=None): + return _match(answers, "PUT", url) + + def delete(url, headers=None): + return _match(answers, "DELETE", url) + + monkeypatch.setattr(requests, "get", get) + monkeypatch.setattr(requests, "post", post) + monkeypatch.setattr(requests, "put", put) + monkeypatch.setattr(requests, "delete", delete) return answers + @pytest.fixture def client(): - app.config['TESTING'] = True + app.config["TESTING"] = True with app.test_client() as client: yield client + +def isdate(dt): + """verify that a string looks like an iso8901 date/time string YYYY-MM-DDTHH:MM:SS.MS""" + return re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[.]\d+$", dt) + + +def isuuid(gu): + """verify that a string looks like a guid""" + return re.match(r"[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}$", gu) + + config.init() + def test_api(client, mockrequests): - dummyflow = {'link': {'href': 'buckets/link1/flows/flow1'}, 'name': 'flowname'} - mockrequests.extend([ - _req('GET', 'http://nifi-registry:18080/nifi-registry-api/buckets', - _resp(200, [{'link': {'href':'buckets/link1'}}])), - _req('GET', 'http://nifi-registry:18080/nifi-registry-api/buckets/link1/flows', - _resp(200, [dummyflow])), - _req('POST', 'http://newtarget1/url/api/graph/main', - _resp(200, {'id':'group1'})) - ]) + env_name = "TEST_API_GRAB_ENVIRON" + os.environ[env_name] = "xyz" + assert config._grab_env(env_name, "foo") == "xyz" + assert config._grab_env(env_name) == "xyz" + del os.environ[env_name] + assert config._grab_env(env_name, "foo") == "foo" + try: + config._grab_env(env_name) + assert not "config._grab_env(env_name) should throw errors.DistributorAPIConfigError" + except errors.DistributorAPIConfigError as e: + # expected result + pass + + dummyflow = {"link": {"href": "buckets/link1/flows/flow1"}, "name": "flowname"} + + nifi_url = "http://nifi-registry:18080/nifi-registry-api" + mockrequests.extend( + [ + _req("GET", nifi_url + "/buckets", _resp(200, [{"link": {"href": "buckets/link1"}}])), + _req("GET", nifi_url + "/buckets/link1/flows", _resp(200, [dummyflow])), + _req("POST", "http://newtarget1/url/api/graph/main", _resp(200, {"id": "group1"})), + _req("GET", "/does/not/exist", _resp(404, [{"link": {"href": "does/not/exist"}}])), + _req( + "GET", + "/distributor/distribution-targets/components?name=foo&version=bar", + _resp(200, {"id": "groupd", "components": [{"componentUrl": "COMPONENTURL"}]}), + ), + _req("GET", "COMPONENTURL", _resp(200, {"id": "groupComponentUrl"})), + _req( + "GET", + "/distributor/distribution-targets/components?name=foo&version=bar2", + _resp(200, {"id": "groupd", "components": None}), + ), + ] + ) for rule in app.url_map.iter_rules(): - print(rule) - url = '/distributor/distribution-targets' - url2 = url + '/notfound' - url3 = url2 + '/process-groups' - assert(len(client.get(url).get_json()['distributionTargets']) == 0) - assert(client.get(url2).status_code == 404) - assert(client.put(url2, json={'name': 'notfound1', 'runtimeApiUrl': 'http://notfound/url'}).status_code == 404) - assert(client.delete(url2).status_code == 404) - assert(client.post(url3, json={'processGroupId': 'group1'}).status_code == 404) - resp = client.post(url, json={'name': 'target1', 'runtimeApiUrl': 'http://target/url'}) - assert(resp.status_code == 200) - print(resp.get_json()) - url2 = '/distributor/distribution-targets/' + resp.get_json()['id'] - url3 = url2 + '/process-groups' - assert(len(client.get(url).get_json()['distributionTargets']) == 1) - assert(client.get(url2).status_code == 200) - assert(client.put(url2, json={'name': 'newtarget1', 'runtimeApiUrl': 'http://newtarget1/url'}).status_code == 200) - assert(client.post(url3, json={'processGroupId': 'group1'}).status_code == 404) - dummyflow['identifier'] = 'group1' - assert(client.post(url3, json={'processGroupId': 'group1'}).status_code == 501) - assert(client.delete(url2).status_code == 200) - assert(client.delete(url2).status_code == 404) + print(rule) + url = "/distributor/distribution-targets" + url2 = url + "/notfound" + url3 = url2 + "/process-groups" + assert len(client.get(url).get_json()["distributionTargets"]) == 0 + assert client.get(url2).status_code == 404 + assert client.put(url2, json={"name": "notfound1", "runtimeApiUrl": "http://notfound/url"}).status_code == 404 + assert client.delete(url2).status_code == 404 + assert client.post(url3, json={"processGroupId": "group1"}).status_code == 404 + resp = client.post(url, json={"name": "target1", "runtimeApiUrl": "http://target/url"}) + assert resp.status_code == 200 + + # print(resp.get_json()) + url2 = "/distributor/distribution-targets/" + resp.get_json()["id"] + url3 = url2 + "/process-groups" + assert len(client.get(url).get_json()["distributionTargets"]) == 1 + + assert client.get(url2).status_code == 200 + assert client.put(url2, json={"name": "newtarget1", "runtimeApiUrl": "http://newtarget1/url"}).status_code == 200 + assert client.put(url2, json={"name": "newtarget1", "runtimeApiUrl": "http://newtarget1/url"}).status_code == 200 + + assert client.post(url3, json={"processGroupId": "group1"}).status_code == 404 + assert client.post(url3, json={"processGroupId": "group1"}).status_code == 404 + dummyflow["identifier"] = "group1" + assert client.post(url3, json={"processGroupId": "group1"}).status_code == 501 + + assert client.delete(url2).status_code == 200 + assert client.delete(url2).status_code == 404 + url4 = "/does/not/exist" + + # the following tests do not require an http client but do use requests lib + + # test get_json() exception case + try: + utils.get_json(url4) + assert not "utils.get_json(url4) should throw errors.DistributorAPIError" + except errors.DistributorAPIError as e: + # expected result + pass + + # _req.SHOWMATCHES = True + ret = onboarding_client.get_components_indexed(url, [("foo", "bar")]) + assert ret == {("foo", "bar"): {"id": "groupComponentUrl"}} + + # + try: + ret = onboarding_client.get_components_indexed(url, [("foo", "bar2")]) + assert ( + not "onboarding_client.get_components_indexed(...foo,bar2) should throw errors.DistributorAPIResourceNotFound" + ) + except errors.DistributorAPIResourceNotFound as e: + # expected result + pass + + +def test_data_access(): + # various tests for data_access.py + + saved_cache = copy.deepcopy(data_access.get_distribution_targets()) + ret = data_access.get_distribution_target("ds") + assert ret == {} + + # new transform_request() + req1 = {"name": "req1", "runtimeApiUrl": "rtau1", "nextDistributionTargetId": "ndti1"} + treq1 = data_access.transform_request(req1) + assert isdate(treq1["created"]) + assert isdate(treq1["modified"]) + assert isuuid(treq1["dt_id"]) + assert treq1["processGroups"] == [] + + # new transform_request() + req2 = {"name": "req2", "runtimeApiUrl": "rtau2", "nextDistributionTargetId": "ndti1"} + treq2 = data_access.transform_request(req2) + assert isdate(treq2["created"]) + assert isdate(treq2["modified"]) + assert isuuid(treq2["dt_id"]) + assert treq2["processGroups"] == [] + + # merge_request() should copy certain values from 2nd arg into 1st arg + ret = data_access.merge_request(treq1, treq2) + assert ret["name"] == treq2["name"] + assert ret["runtimeApiUrl"] == treq2["runtimeApiUrl"] + assert ret["description"] is None + assert ret["nextDistributionTargetId"] == treq2["nextDistributionTargetId"] + + # add_distribution_target() adds to the cache + ret = data_access.add_distribution_target({"dt_id": "dt1", "val": "1", "processGroups": []}) + assert data_access.get_distribution_target("dt1")["val"] == "1" + + # update_distribution_target() updates an existing element of the cache + # If the element exists, it returns True + ret = data_access.update_distribution_target({"dt_id": "dt1", "val": "1b", "processGroups": []}) + assert ret + assert data_access.get_distribution_target("dt1")["val"] == "1b" + + # update_distribution_target() updates an existing element of the cache + # If the element does not exist, it returns False + ret = data_access.update_distribution_target({"dt_id": "dt2", "val": "2", "processGroups": []}) + assert not ret + + # add_process_group adds an element to the processGroups array of the distribution target + # if the element exists, returns true, else false + assert data_access.add_process_group("dt1", {"processed": "p1"}) + assert isdate(data_access.get_distribution_target("dt1")["processGroups"][0]["processed"]) + assert not data_access.add_process_group("dt2", {"processed": "p1"}) + + # clean up the cache + # if the element exists, + assert data_access.delete_distribution_target("dt1") + assert not data_access.delete_distribution_target("dt2") + + assert data_access.get_distribution_targets() == saved_cache + + +def test_transform(): + # various tests for transform.py + flow1 = {"flowContents": {"processors": []}} + flow2 = { + "flowContents": { + "processors": [ + { + "bundle": {"artifact": "artifact1", "version": "version1"}, + } + ] + } + } + flow3 = { + "flowContents": { + "processors": [ + { + "bundle": {"artifact": "artifact1", "version": "version1"}, + }, + {"bundle": {"artifact": "artifact2", "version": "version2"}}, + ] + } + } + assert transform.extract_components_from_flow(flow1) == [] + assert transform.extract_components_from_flow(flow2) == [("artifact1", "version1")] + assert transform.extract_components_from_flow(flow3) == [("artifact1", "version1"), ("artifact2", "version2")] diff --git a/mod/distributorapi/tests/test_registry_client.py b/mod/distributorapi/tests/test_registry_client.py index 3767242..cd80e3a 100644 --- a/mod/distributorapi/tests/test_registry_client.py +++ b/mod/distributorapi/tests/test_registry_client.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,9 +18,7 @@ from distributor import registry_client as rc def test_add_url_from_link(): - test = {"link": {"href": "bar"}, "name": "jane", "age": 33, - "innerTest": {"link": {"href": "baz"}, "name": "bob"} - } + test = {"link": {"href": "bar"}, "name": "jane", "age": 33, "innerTest": {"link": {"href": "baz"}, "name": "bob"}} result = rc._add_url_from_link("http://foo", test) assert result["selfUrl"] == "http://foo/bar" @@ -62,8 +60,7 @@ def test_flow_versions(monkeypatch): print(url) return [] - monkeypatch.setattr(distributor.registry_client, "_get_json", - fake_get_json_many) + monkeypatch.setattr(distributor.registry_client, "_get_json", fake_get_json_many) assert [3, 2, 1] == rc.get_flow_versions("http://registry/buckets/123/flows/abc/") @@ -72,7 +69,6 @@ def test_get_flow_diff_latest(monkeypatch): def fake_get_flow_versions(url): return ["1"] - monkeypatch.setattr(distributor.registry_client, "get_flow_versions", - fake_get_flow_versions) + monkeypatch.setattr(distributor.registry_client, "get_flow_versions", fake_get_flow_versions) assert None == rc.get_flow_diff_latest("http://registry", "http://registry/buckets/123/flows/abc/") diff --git a/mod/distributorapi/tests/test_transform.py b/mod/distributorapi/tests/test_transform.py index f275e63..5ee03ac 100644 --- a/mod/distributorapi/tests/test_transform.py +++ b/mod/distributorapi/tests/test_transform.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ from distributor import transform as tr TEST_DIR = os.path.dirname(__file__) + def _load_data(filename): path = os.path.join(TEST_DIR, filename) with open(path) as f: return json.load(f) + def _setup(): flow = _load_data("flow.json") components = _load_data("components.json") @@ -50,19 +52,18 @@ def test_make_fbp_from_flow(): assert list(sorted(expected)) == list(sorted(actual)) # Test processor to processor scenario - expected = {'metadata': {'data_type': 'json', - 'dmaap_type': 'MR', - 'name': 'foo-conn'}, - 'src': {'node': '75c9a179-b36b-4985-9445-d44c8768d6eb', - 'port': 'ves-pnfRegistration-secondary'}, - 'tgt': {'node': '3fadb641-2079-4ca9-bb07-0df5952967fc', - 'port': 'predict_subscriber'}} + expected = { + "metadata": {"data_type": "json", "dmaap_type": "MR", "name": "foo-conn"}, + "src": {"node": "75c9a179-b36b-4985-9445-d44c8768d6eb", "port": "ves-pnfRegistration-secondary"}, + "tgt": {"node": "3fadb641-2079-4ca9-bb07-0df5952967fc", "port": "predict_subscriber"}, + } actual = [e["payload"] for e in fbp if e["command"] == "addedge"] assert actual[0] == expected or actual[1] == expected # Test input port to processor scenario - expected = {'metadata': {'data_type': 'json', 'dmaap_type': 'MR', - 'name': 'ves-data-conn'}, 'src': {}, - 'tgt': {'node': '75c9a179-b36b-4985-9445-d44c8768d6eb', - 'port': 'ves-notification'}} + expected = { + "metadata": {"data_type": "json", "dmaap_type": "MR", "name": "ves-data-conn"}, + "src": {}, + "tgt": {"node": "75c9a179-b36b-4985-9445-d44c8768d6eb", "port": "ves-notification"}, + } assert actual[0] == expected or actual[1] == expected diff --git a/mod/distributorapi/tests/test_utils.py b/mod/distributorapi/tests/test_utils.py index 481e325..f6530a3 100644 --- a/mod/distributorapi/tests/test_utils.py +++ b/mod/distributorapi/tests/test_utils.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,9 +16,12 @@ from distributor import utils +# more tests are in test_api.py + + def test_urljoin(): assert "http://foo/bar/baz" == utils.urljoin("http://foo", "bar", "baz") assert "http://foo/bar/baz" == utils.urljoin("http://foo/", "bar", "baz") - assert "http://foo/bar/baz?name=some-name&version=1.5.0" \ - == utils.urljoin("http://foo", "bar", "baz", **{"name": "some-name", - "version": "1.5.0"}) + assert "http://foo/bar/baz?name=some-name&version=1.5.0" == utils.urljoin( + "http://foo", "bar", "baz", **{"name": "some-name", "version": "1.5.0"} + ) diff --git a/mod/distributorapi/tox.ini b/mod/distributorapi/tox.ini index 3f8a43b..73721d1 100644 --- a/mod/distributorapi/tox.ini +++ b/mod/distributorapi/tox.ini @@ -1,5 +1,22 @@ +# ============LICENSE_START======================================================= +# Copyright (c) 2022 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + [tox] -envlist = py37,py38 +envlist = py38,py39 +skip_missing_interpreters = true [testenv] deps= diff --git a/mod/distributorapi/version.properties b/mod/distributorapi/version.properties new file mode 100644 index 0000000..194d01c --- /dev/null +++ b/mod/distributorapi/version.properties @@ -0,0 +1,24 @@ +# ============LICENSE_START======================================================= +# Copyright (c) 2020-2022 AT&T Intellectual Property. All rights reserved. + +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# + +major=1 +minor=1 +patch=1 +base_version=${major}.${minor}.${patch} +release_version=${base_version} +snapshot_version=${base_version}-SNAPSHOT diff --git a/mod/genprocessor/Changelog.md b/mod/genprocessor/Changelog.md new file mode 100644 index 0000000..4efd8a1 --- /dev/null +++ b/mod/genprocessor/Changelog.md @@ -0,0 +1,9 @@ +# Change Log +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [1.0.3] - 2022/05/21 + - [DCAEGEN2-3169] - CodeCoverage improvement for dcaegen2-platform-mod-genprocessor + diff --git a/mod/genprocessor/pom.xml b/mod/genprocessor/pom.xml index 22e85f5..a0a3b89 100644 --- a/mod/genprocessor/pom.xml +++ b/mod/genprocessor/pom.xml @@ -2,6 +2,7 @@ <!-- ============LICENSE_START======================================================= Copyright (c) 2020 AT&T Intellectual Property. All rights reserved. +Copyright (C) 2022 Huawei. All rights reserved. ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,7 +27,7 @@ limitations under the License. </parent> <groupId>org.onap.dcaegen2.platform.mod</groupId> <artifactId>genprocessor</artifactId> - <version>1.0.2-SNAPSHOT</version> + <version>1.0.3-SNAPSHOT</version> <name>dcaegen2-platform-mod-genprocessor</name> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> @@ -139,10 +140,10 @@ limitations under the License. </files> </inline> </assembly> - <runCmds> - <runCmd>sed -i /etc/nginx/nginx.conf -e '/^user /d' && touch /var/run/nginx.pid && mkdir -p /www/data && chown -R nginx:nginx /www /etc/nginx /var/cache/nginx /var/log/nginx /var/run/nginx.pid</runCmd> - </runCmds> - <user>nginx</user> + <runCmds> + <runCmd>sed -i /etc/nginx/nginx.conf -e '/^user /d' && touch /var/run/nginx.pid && mkdir -p /www/data && chown -R nginx:nginx /www /etc/nginx /var/cache/nginx /var/log/nginx /var/run/nginx.pid</runCmd> + </runCmds> + <user>nginx</user> <entryPoint> <exec> <arg>nginx</arg> @@ -165,16 +166,16 @@ limitations under the License. <assembly> <descriptorRef>artifact-with-dependencies</descriptorRef> </assembly> - <runCmds> - <runCmd>groupadd -g 1000 dcaemod && useradd --shell /bin/bash -u 1000 -g 1000 -m dcaemod</runCmd> - </runCmds> + <runCmds> + <runCmd>groupadd -g 1000 dcaemod && useradd --shell /bin/bash -u 1000 -g 1000 -m dcaemod</runCmd> + </runCmds> <workdir>/maven</workdir> <env> <GENPROC_WORKING_DIR>/work</GENPROC_WORKING_DIR> <GENPROC_ONBOARDING_API_HOST>http://onboarding-api:8080/onboarding</GENPROC_ONBOARDING_API_HOST> <GENPROC_SLEEP_SEC>10</GENPROC_SLEEP_SEC> </env> - <user>dcaemod</user> + <user>dcaemod</user> <entryPoint> <exec> <arg>java</arg> diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/App.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/App.java index 9c50991..445291e 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/App.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/App.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2022 Huawei. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -205,9 +206,7 @@ public class App { return true; } } - } catch (InterruptedException e) { - throw new RuntimeException("Error while creating jar", e); - } catch (IOException e) { + } catch (InterruptedException | IOException e) { throw new RuntimeException("Error while creating jar", e); } @@ -315,11 +314,11 @@ public class App { String sleepstr = System.getenv("GENPROC_SLEEP_SEC"); long sleepdur = (sleepstr != null)? 1000 * Long.parseLong(sleepstr): 0; do { - try { - main2(args); - } catch (Exception e) { - LOG.error(e.toString(), e); - } + try { + main2(args); + } catch (Exception e) { + LOG.error(e.toString(), e); + } Thread.sleep(sleepdur); } while (sleepdur > 0); return; @@ -401,3 +400,4 @@ public class App { } } } + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Comp.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Comp.java index 69d8776..458f4df 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Comp.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Comp.java @@ -32,4 +32,5 @@ public class Comp { @JsonProperty("selfUrl") public String selfUrl; -}
\ No newline at end of file +} + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompList.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompList.java index fcf25b4..ae15ded 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompList.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompList.java @@ -68,3 +68,4 @@ public class CompList { public List<CompShort> components; } + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompSpec.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompSpec.java index 34b7398..41e7aa5 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompSpec.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/CompSpec.java @@ -142,3 +142,4 @@ public class CompSpec { } } + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/DCAEProcessor.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/DCAEProcessor.java index b920b90..72c0714 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/DCAEProcessor.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/DCAEProcessor.java @@ -89,4 +89,5 @@ public abstract class DCAEProcessor extends AbstractProcessor { return this.relationships; } -}
\ No newline at end of file +} + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/OnboardingAPIClient.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/OnboardingAPIClient.java index 12c3726..85032f7 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/OnboardingAPIClient.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/OnboardingAPIClient.java @@ -62,4 +62,5 @@ public class OnboardingAPIClient { } } -}
\ No newline at end of file +} + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/ProcessorBuilder.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/ProcessorBuilder.java index ca87bda..8d7be7d 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/ProcessorBuilder.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/ProcessorBuilder.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2022 Huawei. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -112,7 +113,7 @@ public class ProcessorBuilder { ccFile.addAttribute(attr); } - private static void addMethod(CtClass target, String methodCode) { + public static void addMethod(CtClass target, String methodCode) { try { CtMethod method = CtMethod.make(methodCode, target); target.addMethod(method); @@ -192,4 +193,5 @@ public class ProcessorBuilder { addMethod(target, createCodeRelationships(compSpec)); } -}
\ No newline at end of file +} + diff --git a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Utils.java b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Utils.java index 9a7c50a..8dbc062 100644 --- a/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Utils.java +++ b/mod/genprocessor/src/main/java/org/onap/dcae/genprocessor/Utils.java @@ -41,4 +41,5 @@ public class Utils { return String.format("%s-%s", compSpec.name, compSpec.version); } -}
\ No newline at end of file +} + diff --git a/mod/genprocessor/src/test/java/sandbox/AppTest.java b/mod/genprocessor/src/test/java/sandbox/AppTest.java index d6a1ca3..a4d58bc 100644 --- a/mod/genprocessor/src/test/java/sandbox/AppTest.java +++ b/mod/genprocessor/src/test/java/sandbox/AppTest.java @@ -1,6 +1,7 @@ /*- * ============LICENSE_START======================================================= * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved. + * Copyright (C) 2022 Huawei. All rights reserved. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +21,7 @@ package sandbox; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.FileWriter; @@ -28,12 +30,16 @@ import java.io.Writer; import java.net.URISyntaxException; +import java.net.URL; +import java.net.URLClassLoader; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; +import javassist.ClassPool; +import javassist.CtClass; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -50,10 +56,13 @@ import org.onap.dcae.genprocessor.CompSpec; import org.onap.dcae.genprocessor.DCAEProcessor; import org.onap.dcae.genprocessor.OnboardingAPIClient; import org.onap.dcae.genprocessor.Utils; +import org.onap.dcae.genprocessor.CompList; +import org.onap.dcae.genprocessor.ProcessorBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Unit test for simple App. */ @@ -78,6 +87,28 @@ public class AppTest { CompSpec cs = new CompSpec(); cs.unpackSelf(mx); assertEquals(Utils.formatNameForJar(cs), "SomeJar-2.0"); + try { + CompSpec.loadComponentSpec(new File("sandbox/temp.txt")); + } catch (RuntimeException e) { + // expected case + return; + } + fail("Exception is not thrown"); + } + + @Test + public void testGetNameForJavaClass() { + CompList.CompShort compShort = new CompList.CompShort(); + compShort.name = "test"; + compShort.getNameForJavaClass(); + compShort.componentUrl = "6:invalidURI"; + try { + compShort.getComponentUrlAsURI(); + } catch (RuntimeException e) { + // expected case + return; + } + fail("Exception is not thrwon"); } @@ -174,5 +205,32 @@ public class AppTest { App.main(new String[] { "load" }); /* gen case */ App.main(new String[] { "gen" }); + + URL[] jarURLs = new URL[1]; + try { + App.loadFromJars(jarURLs); + } catch (NullPointerException e) { + // expected case + return; + } + fail("Exception is not thrown"); + } + + @Test + public void testAddMethod() { + try { + ClassPool pool = ClassPool.getDefault(); + CtClass base = pool.get(DCAEProcessor.class.getName()); + + CtClass cc = pool.makeClass(String.format("org.onap.dcae.%s", DCAEProcessor.class)); + cc.setSuperclass(base); + + ProcessorBuilder.addMethod(cc, "test"); + } catch (Exception e) { + // expected case + return; + } + fail("Exception is not thrown"); } } + diff --git a/mod/genprocessor/version.properties b/mod/genprocessor/version.properties new file mode 100644 index 0000000..39e03d9 --- /dev/null +++ b/mod/genprocessor/version.properties @@ -0,0 +1,6 @@ +major=1
+minor=0
+patch=3
+base_version=${major}.${minor}.${patch}
+release_version=${base_version}
+snapshot_version=${base_version}-SNAPSHOT
diff --git a/mod/runtimeapi/Changelog.md b/mod/runtimeapi/Changelog.md index 23168b8..9f31f72 100644 --- a/mod/runtimeapi/Changelog.md +++ b/mod/runtimeapi/Changelog.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.3.3] - 2022-06-03 +- [DCAEGEN2-3087] Switch helmgen-core version to 1.0.4 (includes fix for log path) + ## [1.3.2] - 2022-01-04 - [DCAEGEN2-3028] Change version format to MMddHHmm (removed YY) - [DCAEGEN2-3008] Update helmgen-core from snapshot to released version diff --git a/mod/runtimeapi/pom.xml b/mod/runtimeapi/pom.xml index 7188527..56a0626 100644 --- a/mod/runtimeapi/pom.xml +++ b/mod/runtimeapi/pom.xml @@ -2,8 +2,7 @@ <!-- ============LICENSE_START======================================================= Copyright (c) 2019-2022 AT&T Intellectual Property. All rights reserved. -================================================================================ -Modifications Copyright (c) 2020 Nokia +Copyright (c) 2020 Nokia ================================================================================ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,114 +18,114 @@ limitations under the License. ============LICENSE_END========================================================= --> <project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> - <modules> - <module>runtime-core</module> - <module>runtime-web</module> - </modules> - <parent> - <groupId>org.onap.oparent</groupId> - <artifactId>oparent</artifactId> - <version>2.0.0</version> - </parent> - <groupId>org.onap.dcaegen2.platform.mod</groupId> - <artifactId>runtimeapi</artifactId> - <version>1.3.2</version> - <name>dcaegen2-platform-mod-runtimeapi</name> - <description>MOD Runtime API</description> - <properties> - <maven.deploy.skip>true</maven.deploy.skip> - <maven.build.timestamp.format>yyyyMMdd'T'HHmmss</maven.build.timestamp.format> - <sonar.coverage.jacoco.xmlReportPaths>${project.reporting.outputDirectory}/jacoco-ut/jacoco.xml - </sonar.coverage.jacoco.xmlReportPaths> - <enforcer.skip>true</enforcer.skip> - </properties> - <profiles> - <profile> - <id>java11</id> - <properties> - <maven.compiler.source>11</maven.compiler.source> - <maven.compiler.target>11</maven.compiler.target> - <maven.compiler.release>11</maven.compiler.release> - </properties> - <activation> - <property> - <name>m2e.version</name> - </property> - <activeByDefault>true</activeByDefault> - </activation> - </profile> - <profile> - <id>java8</id> - <properties> - <maven.compiler.source>1.8</maven.compiler.source> - <maven.compiler.target>1.8</maven.compiler.target> - </properties> - </profile> - </profiles> - <dependencies> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>2.8.0</version> - </dependency> - </dependencies> - <build> - <plugins> - <!-- support sonar in multi-module project --> - <plugin> - <groupId>org.sonarsource.scanner.maven</groupId> - <artifactId>sonar-maven-plugin</artifactId> - <version>3.6.0.1398</version> - </plugin> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <packaging>pom</packaging> + <modules> + <module>runtime-core</module> + <module>runtime-web</module> + </modules> + <parent> + <groupId>org.onap.oparent</groupId> + <artifactId>oparent</artifactId> + <version>2.0.0</version> + </parent> + <groupId>org.onap.dcaegen2.platform.mod</groupId> + <artifactId>runtimeapi</artifactId> + <version>1.3.3</version> + <name>dcaegen2-platform-mod-runtimeapi</name> + <description>MOD Runtime API</description> + <properties> + <maven.deploy.skip>true</maven.deploy.skip> + <maven.build.timestamp.format>yyyyMMdd'T'HHmmss</maven.build.timestamp.format> + <sonar.coverage.jacoco.xmlReportPaths>${project.reporting.outputDirectory}/jacoco-ut/jacoco.xml + </sonar.coverage.jacoco.xmlReportPaths> + <enforcer.skip>true</enforcer.skip> + </properties> + <profiles> + <profile> + <id>java11</id> + <properties> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> + <maven.compiler.release>11</maven.compiler.release> + </properties> + <activation> + <property> + <name>m2e.version</name> + </property> + <activeByDefault>true</activeByDefault> + </activation> + </profile> + <profile> + <id>java8</id> + <properties> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> + </properties> + </profile> + </profiles> + <dependencies> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>2.8.0</version> + </dependency> + </dependencies> + <build> + <plugins> + <!-- support sonar in multi-module project --> + <plugin> + <groupId>org.sonarsource.scanner.maven</groupId> + <artifactId>sonar-maven-plugin</artifactId> + <version>3.6.0.1398</version> + </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-enforcer-plugin</artifactId> - <version>1.4.1</version> - <executions> - <execution> - <id>enforce-no-snapshots</id> - <goals> - <goal>enforce</goal> - </goals> - <configuration> - <rules> - <requireReleaseDeps> - <message>No Snapshots Allowed!</message> - </requireReleaseDeps> - </rules> - <fail>false</fail> - </configuration> - </execution> - </executions> - </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>1.4.1</version> + <executions> + <execution> + <id>enforce-no-snapshots</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireReleaseDeps> + <message>No Snapshots Allowed!</message> + </requireReleaseDeps> + </rules> + <fail>false</fail> + </configuration> + </execution> + </executions> + </plugin> - </plugins> - </build> - <dependencyManagement> - <dependencies> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-parent</artifactId> - <version>2.1.7.RELEASE</version> - <type>pom</type> - <scope>import</scope> - <exclusions> - <exclusion> - <groupId>org.springframework.plugin</groupId> - <artifactId>spring-plugin-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.springframework.plugin</groupId> - <artifactId>spring-plugin-core</artifactId> - <version>2.0.0.RELEASE</version> - </dependency> - </dependencies> - </dependencyManagement> + </plugins> + </build> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-parent</artifactId> + <version>2.1.7.RELEASE</version> + <type>pom</type> + <scope>import</scope> + <exclusions> + <exclusion> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-core</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.springframework.plugin</groupId> + <artifactId>spring-plugin-core</artifactId> + <version>2.0.0.RELEASE</version> + </dependency> + </dependencies> + </dependencyManagement> </project> diff --git a/mod/runtimeapi/runtime-core/pom.xml b/mod/runtimeapi/runtime-core/pom.xml index 5970bfd..3ca24e7 100644 --- a/mod/runtimeapi/runtime-core/pom.xml +++ b/mod/runtimeapi/runtime-core/pom.xml @@ -24,13 +24,13 @@ limitations under the License. <parent> <artifactId>runtimeapi</artifactId> <groupId>org.onap.dcaegen2.platform.mod</groupId> - <version>1.3.2</version> + <version>1.3.3</version> </parent> <modelVersion>4.0.0</modelVersion> <artifactId>runtime-core</artifactId> - <version>1.3.2</version> - + <version>1.3.3</version> + <properties><onap-gerrit-review>-changelog-missing</onap-gerrit-review></properties> <dependencies> <dependency> <groupId>org.springframework.boot</groupId> @@ -50,7 +50,7 @@ limitations under the License. <dependency> <groupId>org.onap.dcaegen2.platform</groupId> <artifactId>helmchartgenerator-core</artifactId> - <version>1.0.3</version> + <version>1.0.4</version> </dependency> <dependency> <groupId>org.yaml</groupId> diff --git a/mod/runtimeapi/runtime-web/pom.xml b/mod/runtimeapi/runtime-web/pom.xml index af89495..01ea8fd 100644 --- a/mod/runtimeapi/runtime-web/pom.xml +++ b/mod/runtimeapi/runtime-web/pom.xml @@ -23,18 +23,19 @@ limitations under the License. <parent> <groupId>org.onap.dcaegen2.platform.mod</groupId> <artifactId>runtimeapi</artifactId> - <version>1.3.2</version> + <version>1.3.3</version> </parent> <artifactId>runtime-web</artifactId> - <version>1.3.2-SNAPSHOT</version> + <version>1.3.3-SNAPSHOT</version> <packaging>jar</packaging> + <properties><onap-gerrit-review>-changelog-missing</onap-gerrit-review></properties> <name>runtime-web</name> <description>MOD Runtime Web Module</description> <dependencies> <dependency> <groupId>org.onap.dcaegen2.platform.mod</groupId> <artifactId>runtime-core</artifactId> - <version>1.3.2</version> + <version>1.3.3</version> </dependency> <dependency> <groupId>org.springframework.boot</groupId> |