Add acumos adapter project

Signed-off-by: Andrew Gauld <agauld@att.com>
Issue-ID: DCAEGEN2-1860
Change-Id: Ib22fd2aa61fe7761bacf85e69540d11803c7acee
Signed-off-by: Andrew Gauld <agauld@att.com>

1 files changed, 155 insertions, 0 deletions

diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py
new file mode 100644
index 0000000..a5ead96
--- /dev/null
+++ b/adapter/acumos/aoconversion/dataformat_gen.py
@@ -0,0 +1,155 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+from subprocess import PIPE, Popen
+import json
+from jsonschema import validate
+import requests
+from aoconversion import utils, exceptions
+
+
+def _get_js_schema():
+    res = requests.get("http://json-schema.org/draft-04/schema#")
+    return res.json()
+
+
+def _get_dcae_df_schema():
+    res = requests.get(
+        "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD"
+    )
+    return res.json()
+
+
+def _protobuf_to_js(proto_path):
+    """
+    Converts a protobuf to jsonschema and returns the generated schema as a JSON object.
+    """
+    cmd = ["protobuf-jsonschema", proto_path]
+    p = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    out = p.stdout.read()
+    asjson = json.loads(out)
+
+    # change the defintion names to remove the random package name that acumos generates
+    defs = asjson["definitions"]
+    defns = list(defs.keys())
+    for defn in defns:
+        # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key
+        defs[defn.split(".")[1]] = defs.pop(defn)
+
+    # make sure what we got out is a valid jsonschema
+    draft4 = _get_js_schema()
+    validate(instance=asjson, schema=draft4)
+
+    return asjson
+
+
+def _get_needed_formats(meta):
+    """
+    Read the metadata and figure out what the principle data formats are.
+    We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another.
+    We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format
+    """
+    # we use a dict because multiple methods may reuse names
+    needed_formats = {}
+    for method in meta["methods"]:
+        needed_formats[meta["methods"][method]["input"]] = 1
+        needed_formats[meta["methods"][method]["output"]] = 1
+    return list(needed_formats.keys())
+
+
+def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema):
+    """
+    Generates a collection of data formats from the model .proto
+    This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters,
+    but generate_dcae_data_formats requires some mocking etc.
+    """
+    js = _protobuf_to_js(proto_path)
+    needed_formats = _get_needed_formats(meta)
+
+    data_formats = []
+
+    used_defns = []
+
+    # iterate over and convert
+    for nf in needed_formats:
+        defn = js["definitions"][nf]
+
+        definitions = {}
+
+        # check for the case where we have an array of other defns
+        for prop in defn["properties"]:
+            if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]:
+                unclean_ref_name = defn["properties"][prop]["items"]["$ref"]
+                clean_ref_name = unclean_ref_name.split(".")[1]
+                if clean_ref_name in js["definitions"]:
+                    defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name)
+                    definitions[clean_ref_name] = js["definitions"][clean_ref_name]
+                    used_defns.append(clean_ref_name)
+                else:  # this is bad/unsupported, investigate
+                    raise exceptions.UnsupportedFormatScenario()
+
+        # the defns created by this tool do not include a schema field.
+        # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12
+        defn["$schema"] = "http://json-schema.org/draft-04/schema#"
+
+        # Include the definitions, which may be empty {}
+        defn["definitions"] = definitions
+
+        # Validate that our resulting jsonschema is valid jsonschema
+        validate(instance=defn, schema=draft_4_schema)
+
+        # we currently hardcode dataformatversion, since it is the latest and has been for years  https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD
+        dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn}
+
+        # make sure the schema validates against the DCAE data format schema
+        validate(instance=dcae_df, schema=dcae_df_schema)
+
+        # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats
+        data_formats.append(dcae_df)
+
+    # make sure every definitin we got out was used. Otherwise, this requires investigation!!
+    if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())):
+        raise exceptions.UnsupportedFormatScenario()
+
+    return data_formats
+
+
+# Public
+
+
+def generate_dcae_data_formats(model_repo_path, model_name):
+    """
+    Generates a collection of data formats from the model .proto
+    Writes them to disk
+    Returns them as the return of this call so this can be fed directly into spec gen
+    """
+    data_formats = _generate_dcae_data_formats(
+        "{0}/{1}/model.proto".format(model_repo_path, model_name),
+        utils.get_metadata(model_repo_path, model_name),
+        _get_dcae_df_schema(),
+        _get_js_schema(),
+    )
+
+    # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that
+    for df in data_formats:
+        # name_version seems like a reasonable filename
+        fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"])
+        with open("{0}/{1}".format(model_repo_path, fname), "w") as f:
+            f.write(json.dumps(df))
+
+    return data_formats