diff options
Diffstat (limited to 'adapter/acumos/aoconversion/dataformat_gen.py')
| -rw-r--r-- | adapter/acumos/aoconversion/dataformat_gen.py | 155 |
1 files changed, 155 insertions, 0 deletions
diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py new file mode 100644 index 0000000..a5ead96 --- /dev/null +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -0,0 +1,155 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from subprocess import PIPE, Popen +import json +from jsonschema import validate +import requests +from aoconversion import utils, exceptions + + +def _get_js_schema(): + res = requests.get("http://json-schema.org/draft-04/schema#") + return res.json() + + +def _get_dcae_df_schema(): + res = requests.get( + "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD" + ) + return res.json() + + +def _protobuf_to_js(proto_path): + """ + Converts a protobuf to jsonschema and returns the generated schema as a JSON object. + """ + cmd = ["protobuf-jsonschema", proto_path] + p = Popen(cmd, stderr=PIPE, stdout=PIPE) + out = p.stdout.read() + asjson = json.loads(out) + + # change the defintion names to remove the random package name that acumos generates + defs = asjson["definitions"] + defns = list(defs.keys()) + for defn in defns: + # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key + defs[defn.split(".")[1]] = defs.pop(defn) + + # make sure what we got out is a valid jsonschema + draft4 = _get_js_schema() + validate(instance=asjson, schema=draft4) + + return asjson + + +def _get_needed_formats(meta): + """ + Read the metadata and figure out what the principle data formats are. + We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another. + We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format + """ + # we use a dict because multiple methods may reuse names + needed_formats = {} + for method in meta["methods"]: + needed_formats[meta["methods"][method]["input"]] = 1 + needed_formats[meta["methods"][method]["output"]] = 1 + return list(needed_formats.keys()) + + +def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema): + """ + Generates a collection of data formats from the model .proto + This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters, + but generate_dcae_data_formats requires some mocking etc. + """ + js = _protobuf_to_js(proto_path) + needed_formats = _get_needed_formats(meta) + + data_formats = [] + + used_defns = [] + + # iterate over and convert + for nf in needed_formats: + defn = js["definitions"][nf] + + definitions = {} + + # check for the case where we have an array of other defns + for prop in defn["properties"]: + if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]: + unclean_ref_name = defn["properties"][prop]["items"]["$ref"] + clean_ref_name = unclean_ref_name.split(".")[1] + if clean_ref_name in js["definitions"]: + defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name) + definitions[clean_ref_name] = js["definitions"][clean_ref_name] + used_defns.append(clean_ref_name) + else: # this is bad/unsupported, investigate + raise exceptions.UnsupportedFormatScenario() + + # the defns created by this tool do not include a schema field. + # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12 + defn["$schema"] = "http://json-schema.org/draft-04/schema#" + + # Include the definitions, which may be empty {} + defn["definitions"] = definitions + + # Validate that our resulting jsonschema is valid jsonschema + validate(instance=defn, schema=draft_4_schema) + + # we currently hardcode dataformatversion, since it is the latest and has been for years https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD + dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn} + + # make sure the schema validates against the DCAE data format schema + validate(instance=dcae_df, schema=dcae_df_schema) + + # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats + data_formats.append(dcae_df) + + # make sure every definitin we got out was used. Otherwise, this requires investigation!! + if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())): + raise exceptions.UnsupportedFormatScenario() + + return data_formats + + +# Public + + +def generate_dcae_data_formats(model_repo_path, model_name): + """ + Generates a collection of data formats from the model .proto + Writes them to disk + Returns them as the return of this call so this can be fed directly into spec gen + """ + data_formats = _generate_dcae_data_formats( + "{0}/{1}/model.proto".format(model_repo_path, model_name), + utils.get_metadata(model_repo_path, model_name), + _get_dcae_df_schema(), + _get_js_schema(), + ) + + # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that + for df in data_formats: + # name_version seems like a reasonable filename + fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"]) + with open("{0}/{1}".format(model_repo_path, fname), "w") as f: + f.write(json.dumps(df)) + + return data_formats |