diff options
Diffstat (limited to 'adapter/acumos/aoconversion')
| -rw-r--r-- | adapter/acumos/aoconversion/__init__.py | 0 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/adapter.py | 50 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/convert.py | 33 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/dataformat_gen.py | 155 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/docker_gen.py | 110 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/exceptions.py | 29 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/index.html | 254 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/scanner.py | 312 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/spec_gen.py | 112 | ||||
| -rw-r--r-- | adapter/acumos/aoconversion/utils.py | 24 |
10 files changed, 1079 insertions, 0 deletions
diff --git a/adapter/acumos/aoconversion/__init__.py b/adapter/acumos/aoconversion/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/adapter/acumos/aoconversion/__init__.py diff --git a/adapter/acumos/aoconversion/adapter.py b/adapter/acumos/aoconversion/adapter.py new file mode 100644 index 0000000..38b5a71 --- /dev/null +++ b/adapter/acumos/aoconversion/adapter.py @@ -0,0 +1,50 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Command line for Acumos -> ONAP adapter +""" + +import os +import shutil +import sys +import yaml + + +def adapter(): + """ + Run the adapter + """ + import warnings + warnings.simplefilter('ignore') + with open(sys.argv[1], 'r') as f: + ycfg = yaml.safe_load(f.read()) + from aoconversion import scanner + config = scanner.Config(**ycfg) + try: + shutil.rmtree(config.tmpdir) + except Exception: + pass + os.makedirs(config.tmpdir) + if config.port: + print('Starting web server') + scanner.serve(config) + else: + print('Starting scan') + scanner.scan(config) + print('Scan complete. Sleeping') diff --git a/adapter/acumos/aoconversion/convert.py b/adapter/acumos/aoconversion/convert.py new file mode 100644 index 0000000..e41820a --- /dev/null +++ b/adapter/acumos/aoconversion/convert.py @@ -0,0 +1,33 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Module that converts acumos models to dcae artifacts +""" +from aoconversion import docker_gen, dataformat_gen, spec_gen + + +def gen_dcae_artifacts_for_model(config, model_name, model_version="latest"): + """ + Generate all dcae artifacts given an acumos model + """ + model_repo_path = config.tmpdir + docker_uri = docker_gen.build_and_push_docker(config, model_name, model_version) + data_formats = dataformat_gen.generate_dcae_data_formats(model_repo_path, model_name) + spec = spec_gen.generate_spec(model_repo_path, model_name, data_formats, docker_uri) + return docker_uri, data_formats, spec diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py new file mode 100644 index 0000000..a5ead96 --- /dev/null +++ b/adapter/acumos/aoconversion/dataformat_gen.py @@ -0,0 +1,155 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +from subprocess import PIPE, Popen +import json +from jsonschema import validate +import requests +from aoconversion import utils, exceptions + + +def _get_js_schema(): + res = requests.get("http://json-schema.org/draft-04/schema#") + return res.json() + + +def _get_dcae_df_schema(): + res = requests.get( + "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD" + ) + return res.json() + + +def _protobuf_to_js(proto_path): + """ + Converts a protobuf to jsonschema and returns the generated schema as a JSON object. + """ + cmd = ["protobuf-jsonschema", proto_path] + p = Popen(cmd, stderr=PIPE, stdout=PIPE) + out = p.stdout.read() + asjson = json.loads(out) + + # change the defintion names to remove the random package name that acumos generates + defs = asjson["definitions"] + defns = list(defs.keys()) + for defn in defns: + # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key + defs[defn.split(".")[1]] = defs.pop(defn) + + # make sure what we got out is a valid jsonschema + draft4 = _get_js_schema() + validate(instance=asjson, schema=draft4) + + return asjson + + +def _get_needed_formats(meta): + """ + Read the metadata and figure out what the principle data formats are. + We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another. + We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format + """ + # we use a dict because multiple methods may reuse names + needed_formats = {} + for method in meta["methods"]: + needed_formats[meta["methods"][method]["input"]] = 1 + needed_formats[meta["methods"][method]["output"]] = 1 + return list(needed_formats.keys()) + + +def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema): + """ + Generates a collection of data formats from the model .proto + This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters, + but generate_dcae_data_formats requires some mocking etc. + """ + js = _protobuf_to_js(proto_path) + needed_formats = _get_needed_formats(meta) + + data_formats = [] + + used_defns = [] + + # iterate over and convert + for nf in needed_formats: + defn = js["definitions"][nf] + + definitions = {} + + # check for the case where we have an array of other defns + for prop in defn["properties"]: + if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]: + unclean_ref_name = defn["properties"][prop]["items"]["$ref"] + clean_ref_name = unclean_ref_name.split(".")[1] + if clean_ref_name in js["definitions"]: + defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name) + definitions[clean_ref_name] = js["definitions"][clean_ref_name] + used_defns.append(clean_ref_name) + else: # this is bad/unsupported, investigate + raise exceptions.UnsupportedFormatScenario() + + # the defns created by this tool do not include a schema field. + # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12 + defn["$schema"] = "http://json-schema.org/draft-04/schema#" + + # Include the definitions, which may be empty {} + defn["definitions"] = definitions + + # Validate that our resulting jsonschema is valid jsonschema + validate(instance=defn, schema=draft_4_schema) + + # we currently hardcode dataformatversion, since it is the latest and has been for years https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD + dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn} + + # make sure the schema validates against the DCAE data format schema + validate(instance=dcae_df, schema=dcae_df_schema) + + # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats + data_formats.append(dcae_df) + + # make sure every definitin we got out was used. Otherwise, this requires investigation!! + if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())): + raise exceptions.UnsupportedFormatScenario() + + return data_formats + + +# Public + + +def generate_dcae_data_formats(model_repo_path, model_name): + """ + Generates a collection of data formats from the model .proto + Writes them to disk + Returns them as the return of this call so this can be fed directly into spec gen + """ + data_formats = _generate_dcae_data_formats( + "{0}/{1}/model.proto".format(model_repo_path, model_name), + utils.get_metadata(model_repo_path, model_name), + _get_dcae_df_schema(), + _get_js_schema(), + ) + + # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that + for df in data_formats: + # name_version seems like a reasonable filename + fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"]) + with open("{0}/{1}".format(model_repo_path, fname), "w") as f: + f.write(json.dumps(df)) + + return data_formats diff --git a/adapter/acumos/aoconversion/docker_gen.py b/adapter/acumos/aoconversion/docker_gen.py new file mode 100644 index 0000000..bf54cfc --- /dev/null +++ b/adapter/acumos/aoconversion/docker_gen.py @@ -0,0 +1,110 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import os +from docker import APIClient +from aoconversion import exceptions, utils + + +def _generate_dockerfile(meta, model_name): + """ + bind the templated docker string + """ + docker_template = """ + FROM python:{VERSION} + + ENV MODELNAME {MODELNAME} + RUN mkdir /app + WORKDIR /app + + ADD ./{MODELNAME} /app/{MODELNAME} + ADD ./requirements.txt /app + + RUN pip install -r /app/requirements.txt && \ + pip install acumos_dcae_model_runner + + ENV DCAEPORT=10000 + EXPOSE $DCAEPORT + + ENTRYPOINT ["acumos_dcae_model_runner"] + CMD ["/app/{MODELNAME}"] + """ + python_version = meta["runtime"]["version"] + return docker_template.format(VERSION=python_version, MODELNAME=model_name) + + +# Public + + +def build_and_push_docker(config, model_name, model_version="latest"): + """ + build and push the dcae docker container + Returns the docker uri so this can be pipelined into specgen + """ + model_repo_path = config.tmpdir + dockerfile_path = "{0}/Dockerfile".format(model_repo_path) + reqs_path = "{0}/requirements.txt".format(model_repo_path) + + # get the metadata + meta = utils.get_metadata(model_repo_path, model_name) + + # write the reqs file, will be removed later + reqs = meta["runtime"]["dependencies"]["pip"]["requirements"] + with open(reqs_path, "w") as f: + for r in reqs: + f.write("{0}=={1}\n".format(r["name"], r["version"])) + + # generate the dockerfile + dockerfile = _generate_dockerfile(meta, model_name) + + # write the dockerfile, will be removed later + with open("{0}/Dockerfile".format(model_repo_path), "w") as f: + f.write(dockerfile) + + docker_uri = "{0}/{1}:{2}".format(config.dockerregistry, model_name, model_version) + + # do the docker build + cli = APIClient(base_url=config.dockerhost, user_agent="Docker-Client-xx.yy") + response = [line.decode() for line in cli.build(path=model_repo_path, rm=True, tag=docker_uri)] + + # clean up the files + os.remove(dockerfile_path) + os.remove(reqs_path) + + # parse the Docker response to see whether we succeeded + for r in response: + # In some scenarios, for example a non-existing Dockerfile, docker build raises a native exception, see: + # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build + # However, if something fails such as a non-existing directory referenced in the dockerfile, NO exception is raised. + # In this case, one of the console output lines is "errorDetail".... + if "errorDetail" in r: + raise exceptions.DockerBuildFail(r) + + # if the above succeeded, we can push + # push. same problem as above; e.g., "no basic auth credentials" does not throw an exception! + response = [ + line.decode() + for line in cli.push( + repository=docker_uri, auth_config={"username": config.dockeruser, "password": config.dockerpass()}, stream=True + ) + ] + for r in response: + if "errorDetail" in r: + raise exceptions.DockerPushFail(r) + + return docker_uri diff --git a/adapter/acumos/aoconversion/exceptions.py b/adapter/acumos/aoconversion/exceptions.py new file mode 100644 index 0000000..8e87458 --- /dev/null +++ b/adapter/acumos/aoconversion/exceptions.py @@ -0,0 +1,29 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + + +class DockerBuildFail(BaseException): + pass + + +class DockerPushFail(BaseException): + pass + + +class UnsupportedFormatScenario(BaseException): + pass diff --git a/adapter/acumos/aoconversion/index.html b/adapter/acumos/aoconversion/index.html new file mode 100644 index 0000000..c94621a --- /dev/null +++ b/adapter/acumos/aoconversion/index.html @@ -0,0 +1,254 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0//EN"> +<!-- +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= +--> +<html> +<meta http-equiv="X-UA-Compatible" content="IE=Edge;chrome=1"/> +<head><title>Acumos-ONAP Adapter Demo</title> +<style> +body { + background-size: cover; + font-family: Arial; + color: blue; + background-color: white; +} +ul { + margin: 0; + padding: 0; + list-style-type: none; +} +ul li { + float: left; + width: 200px; + height: 50px; + background-color: black; + opacity: 1; + line-height: 50px; + text-align: center; + font-size: 14px; + font-weight: 600; + text-transform: uppercase; +} +ul li a { + text-decoration: none; + color: white; + display: block; +} +ul li a:hover { + background-color: green; +} +ul li ul li { + display: none; +} +ul li:hover ul li { + display: block; +} +.navBar { + height: 50px; + background-color: black; +} +.auxNav { + float: right; +} +#itAcumos { + display: none; +} +#onboardingInProgress { + display: none; +} +.form-popup { + display: none; + position: fixed; + top: 100px; + left: 100px; + border: 3px solid #f1f1f1; + z-index: 9; +} +.form-container { + max-width: 350px; + padding: 10px; + background-color: white; +} +.form-container input[type=text], .form-container input[type=password] { + width: 300px; + padding: 15px; + margin: 5px 0 22px 0; + border: none; + background: #f1f1f1; +} +</style> +</head><body> +<div class="navBar"> +<ul> +<li><a>File</a> + <ul> + <li><a onclick="openImportPopup()">Import ...</a></li> + </ul> +</li> +<li class="auxNav"><a>Signout</a></li> +</ul> +</div> +<form class="form-popup form-container" id="importPopup"> + <h2 align="center">Import Model</h2> + <select id="importType" onchange="setImportType()"> + <option value="-" selected="true">-- Choose Import Type --</option> + <option value="itAcumos" >Acumos</option> + </select> + <div id="itAcumos"> + <hr/> + <label for="furl"><b>Acumos Federation URL</b></label> <input id="furl" type="text" name="furl" placeholder="https://server:9084" required> + <button type="button" onclick="lookupCatalogs()">Lookup</button> + </div> + <div id="cAcumos"> + <hr/> + <label for="catMenu"><b>Select Catalog</b></label> <select id="catMenu" onchange="chooseCatalog()"> + <option value="*">All Catalogs</option> + </select> + </div> + <div id="acSols"> + <hr/> + <label for="solMenu"><b>Select Solution</b></label> <select id="solMenu" onchange="chooseSolution()"> + <option value="*">All Solutions</option> + </select> + </div> + <div id="acRevs"> + <hr/> + <label for="revMenu"><b>Select Revision</b></label> <select id="revMenu"> + <option value="*">All Revisions</option> + </select> + </div> + <hr/> + <button type="button" onclick="closeImportPopup()">Cancel</button> + <button id="onboard" type="button" onclick="onBoard()">Onboard</button> + <b id="onboardingInProgress">Onboarding - Please Wait ...</b> +</form> +<script> +function fcomp(n) { + return document.getElementById(n); +} +function cvalue(n) { + return fcomp(n).value; +} + +function uecvalue(n) { + return encodeURIComponent("" + cvalue(n)); +} + +function esc(s) { + return s.replace(/&/g,'&').replace(/</g, '<').replace(/>/g, '>'); +} + +function onBoard() { + fcomp("onboardingInProgress").style.display = "block"; + var url = "/onboard.js?acumos=" + uecvalue("furl"); + if (cvalue("catMenu") != "*") { + url += "&catalogId=" + uecvalue("catMenu"); + if (cvalue("solMenu") != "*") { + url += "&solutionId=" + uecvalue("solMenu"); + if (cvalue("revMenu") != "*") { + url += "&revisionId=" + uecvalue("revMenu"); + } + } + } + let xhr = new XMLHttpRequest(); + xhr.onerror = xhr.onload = function() { + fcomp("onboardingInProgress").style.display = "none"; + } + xhr.open("POST", url); + xhr.send(); +} + +function chooseSolution() { + if (cvalue("solMenu") == "*") { + updatevis(); + } else { + lookupItem("acRevs", "revMenu", "/listRevisions.js?acumos=" + uecvalue("furl") + "&solutionId=" + uecvalue("solMenu")); + } +} +function chooseCatalog() { + if (cvalue("catMenu") == "*") { + updatevis(); + } else { + lookupItem("acSols", "solMenu", "/listSolutions.js?acumos=" + uecvalue("furl") + "&catalogId=" + uecvalue("catMenu")); + } +} +function lookupCatalogs() { + fcomp("onboard").style.display = "block"; + lookupItem("cAcumos", "catMenu", "/listCatalogs.js?acumos=" + uecvalue("furl")); +} +function lookupItem(dblock, smenu, url) { + fcomp(dblock).style.display = "block"; + let xhr = new XMLHttpRequest(); + let catmenu = fcomp(smenu); + catmenu.options.length = 1; + catmenu.options[0].selected = true; + xhr.onload = function() { + let catresp = JSON.parse(this.response); + var i; + for (i = 0; i < catresp.length; i++) { + var option = document.createElement("option"); + option.text = esc(catresp[i].name); + option.value = catresp[i].id; + catmenu.add(option); + } + updatevis(); + }; + xhr.open("GET", url); + xhr.send(); +} + +function updatevis() { + if (cvalue("importType") != "itAcumos") { + fcomp("itAcumos").style.display = "none"; + fcomp("furl").value = ""; + } + if (cvalue("furl") == "") { + fcomp("cAcumos").style.display = "none"; + fcomp("onboard").style.display = "none"; + fcomp("catMenu").options[0].selected = true; + } + if (cvalue("catMenu") == "*") { + fcomp("acSols").style.display = "none"; + fcomp("solMenu").options[0].selected = true; + } + if (cvalue("solMenu") == "*") { + fcomp("acRevs").style.display = "none"; + fcomp("revMenu").options[0].selected = true; + } +} + +function setImportType() { + let di = fcomp("itAcumos"); + if (cvalue("importType") == "itAcumos") { + fcomp("furl").value = ""; + di.style.display = "block"; + } + updatevis(); +} +function openImportPopup() { + fcomp("importType").options[0].selected = true; + fcomp("importPopup").style.display = "block"; + updatevis(); +} +function closeImportPopup() { + fcomp("importPopup").style.display = "none"; +} +</script> +</body> +</html> diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py new file mode 100644 index 0000000..22a5922 --- /dev/null +++ b/adapter/acumos/aoconversion/scanner.py @@ -0,0 +1,312 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json +import os +from pkg_resources import resource_string +import shutil +import traceback +try: + from urllib.parse import unquote_plus + from socketserver import ThreadingMixIn + from http.server import BaseHTTPRequestHandler, HTTPServer +except ImportError: + from urllib import unquote_plus + from SocketServer import ThreadingMixIn + from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer + +import requests + +from aoconversion import convert + + +def _derefconfig(value): + if value.startswith('@'): + with open(value[1:], 'r') as f: + return f.readline().strip() + return value + + +class Config(object): + """ + Configuration parameters as attributes, make sure the required ones are there, + populate defaults. + """ + def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='unix:///var/run/docker.sock', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras): + self.dcaeurl = dcaeurl + self.dcaeuser = dcaeuser + + def x(fmt, *args, **kwargs): + return onboardingurl + fmt.format(*args, **kwargs) + self.oburl = x + self._onboardingpass = onboardingpass + self._onboardinguser = onboardinguser + self.acumosurl = acumosurl + self.certfile = certfile + self.certverify = certverify + self.dockerhost = dockerhost + self.dockerregistry = dockerregistry + self.dockeruser = dockeruser + self._dockerpass = dockerpass + self.interval = interval + self.tmpdir = tmpdir + if catalogs is not None and type(catalogs) is not list: + catalogs = [catalogs] + self.catalogs = catalogs + self.port = port + + def obauth(self): + return (self._onboardinguser, _derefconfig(self._onboardingpass)) + + def dockerpass(self): + return _derefconfig(self._dockerpass) + + +class _AcumosAccess(object): + def __init__(self, config, url): + self.cert = config.certfile + self.verify = config.certverify + self.url = url.strip().rstrip('/') + + def artgetter(self, xrev, matcher): + for art in xrev['artifacts']: + if matcher(art): + def ret(): + nurl = '{}/artifacts/{}/content'.format(self.url, art['artifactId']) + resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify) + if resp.status_code == 500: + resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify) + resp.raise_for_status() + return resp + return ret + return None + + def jsonget(self, format, *args, **kwargs): + nurl = self.url + format.format(*args, **kwargs) + resp = requests.get(nurl, cert=self.cert, verify=self.verify) + if resp.status_code == 500: + resp = requests.get(nurl, cert=self.cert, verify=self.verify) + resp.raise_for_status() + return resp.json()['content'] + + +def _x_proto_matcher(art): + """ Is this artifact the x.proto file? """ + return art['name'].endswith('.proto') + + +def _x_zip_matcher(art): + """ Is this artifact the x.zip file? """ + return art['name'].endswith('.zip') + + +def _md_json_matcher(art): + """ Is this artifact the metadata.json file? """ + return art['name'].endswith('.json') + + +def _walk(config): + """ + Walk the Federation E5 interface of an Acumos instance + """ + url = config.acumosurl + callback = _makecallback(config) + catalogs = config.catalogs + aa = _AcumosAccess(config, url) + for catalog in aa.jsonget('/catalogs'): + if catalogs is not None and catalog['catalogId'] not in catalogs and catalog['name'] not in catalogs: + continue + for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + + +def onboard(aa, callback, solution, revid): + xrev = aa.jsonget('/solutions/{}/revisions/{}', solution['solutionId'], revid) + callback(model_name=solution['name'], model_version=xrev['version'], model_last_updated=xrev['modified'], rating=solution['ratingAverageTenths'] / 10.0, proto_getter=aa.artgetter(xrev, _x_proto_matcher), zip_getter=aa.artgetter(xrev, _x_zip_matcher), metadata_getter=aa.artgetter(xrev, _md_json_matcher)) + + +def _pullfile(source, dest): + with open(dest, 'wb') as f: + for chunk in source().iter_content(65536): + f.write(chunk) + + +_loadedformats = set() +_loadedcomponents = set() + + +def scan(config): + _walk(config) + + +def _makecallback(config): + workdir = config.tmpdir + obauth = config.obauth() + oburl = config.oburl + + def callback(model_name, model_version, model_last_updated, rating, proto_getter, zip_getter, metadata_getter): + model_name = model_name.lower() + model_version = model_version.lower() + compid = (model_name, model_version) + if compid in _loadedcomponents: + print('Skipping component {}: already analyzed'.format(compid)) + return + if proto_getter is None or zip_getter is None or metadata_getter is None: + print('Skipping component {}: does not have required artifacts'.format(compid)) + _loadedcomponents.add(compid) + return + modeldir = '{}/{}'.format(workdir, model_name) + shutil.rmtree(modeldir, True) + os.makedirs(modeldir) + try: + _pullfile(proto_getter, '{}/model.proto'.format(modeldir)) + _pullfile(zip_getter, '{}/model.zip'.format(modeldir)) + _pullfile(metadata_getter, '{}/metadata.json'.format(modeldir)) + except Exception: + print('Skipping component {}: artifact access error'.format(compid)) + _loadedcomponents.add(compid) + return + try: + docker_uri, data_formats, spec = convert.gen_dcae_artifacts_for_model(config, model_name, model_version) + shutil.rmtree(modeldir) + except Exception: + print('Error analyzing artifacts for {}'.format(compid)) + traceback.print_exc() + return + for data_format in data_formats: + fmtid = (data_format['self']['name'], data_format['self']['version']) + if fmtid in _loadedformats: + print('Skipping data format {}: already analyzed'.format(fmtid)) + continue + try: + resp = requests.post(oburl('/dataformats'), json={'owner': config.dcaeuser, 'spec': data_format}, auth=obauth) + if resp.status_code == 409: + print('Skipping data format {}: previously loaded'.format(fmtid)) + _loadedformats.add(fmtid) + continue + resp.raise_for_status() + requests.patch(resp.json()['dataFormatUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status() + print('Loaded data format {}'.format(fmtid)) + _loadedformats.add(fmtid) + except Exception: + print('Error loading data format {}'.format(fmtid)) + traceback.print_exc() + raise + try: + resp = requests.post(oburl('/components'), json={'owner': config.dcaeuser, 'spec': spec}, auth=obauth) + if resp.status_code == 409: + print('Skipping component {}: previously loaded'.format(compid)) + _loadedcomponents.add(compid) + return + resp.raise_for_status() + requests.patch(resp.json()['componentUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status() + print('Loaded component {}'.format(compid)) + _loadedcomponents.add(compid) + except Exception: + print('Error loading component {}'.format(compid)) + traceback.print_exc() + raise + return callback + + +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + protocol_version = "HTTP/1.1" + + +class Apihandler(BaseHTTPRequestHandler): + def doqp(self): + self.qparams = {} + if not self.path or '?' not in self.path: + return + self.path, qp = self.path.split('?', 1) + for x in qp.split('&'): + k, v = x.split('=', 1) + self.qparams[unquote_plus(k)] = unquote_plus(v) + + def replyjson(self, body, ctype='application/json'): + self.replyraw(json.dumps(body).encode('utf-8'), ctype) + + def replyraw(self, data, ctype): + self.send_response(200) + self.send_header('Content-Type', ctype) + self.send_header('Content-Length', len(data)) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + self.doqp() + if self.path == '/' or self.path == '/index.html': + self.replyraw(self.server.index, 'text/html') + return + if 'acumos' not in self.qparams: + self.send_error(400) + return + aa = _AcumosAccess(self.server.config, self.qparams['acumos']) + if self.path == '/listCatalogs.js': + self.replyjson([{'name': x['name'], 'id': x['catalogId']} for x in aa.jsonget('/catalogs')]) + return + if self.path == '/listSolutions.js': + if 'catalogId' not in self.qparams: + self.send_error(400) + return + self.replyjson([{'name': x['name'], 'id': x['solutionId']} for x in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId'])]) + return + if self.path == '/listRevisions.js': + if 'solutionId' not in self.qparams: + self.send_error(400) + return + self.replyjson([{'name': x['version'], 'id': x['revisionId']} for x in aa.jsonget('/solutions/{}/revisions', self.qparams['solutionId'])]) + return + self.send_error(404) + + def do_POST(self): + self.doqp() + if self.path == '/onboard.js': + if 'acumos' not in self.qparams: + self.send_error(400) + return + aa = _AcumosAccess(self.server.config, self.qparams['acumos']) + callback = self.server.callback + if 'catalogId' not in self.qparams: + for catalog in aa.jsonget('/catalogs'): + for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + elif 'solutionId' not in self.qparams: + for solution in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId']): + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + elif 'revisionId' not in self.qparams: + solution = aa.jsonget('/solutions/{}', self.qparams['solutionId']) + for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']): + onboard(aa, callback, solution, revision['revisionId']) + else: + solution = aa.jsonget('/solutions/{}', self.qparams['solutionId']) + onboard(aa, callback, solution, self.qparams['revisionId']) + self.replyraw('OK', 'text/plain') + return + self.send_error(400) + + +def serve(config): + server = ThreadedHTTPServer(('', config.port), Apihandler) + server.config = config + server.callback = _makecallback(config) + server.index = resource_string(__name__, 'index.html') + server.serve_forever() diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py new file mode 100644 index 0000000..1662872 --- /dev/null +++ b/adapter/acumos/aoconversion/spec_gen.py @@ -0,0 +1,112 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +""" +Generates DCAE component specs +""" + + +import json +from jsonschema import validate +import requests +from aoconversion import utils + + +def _get_dcae_cs_schema(): + res = requests.get( + "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD" + ) + return res.json() + + +def _get_format_version(target_name, data_formats): + """ + search through the data formats for name, make sure we have it, and retrieve the version + """ + # the df must exist, since the data formats were generated from the same metadata, or dataformats call blew up. + # So we don't do error checking here + for df in data_formats: + if df["self"]["name"] == target_name: + return df["self"]["version"] + + +def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri): + """ + Function that generates the component spec from the model metadata and docker info + Broken out to be unit-testable. + """ + + spec = { + "self": { + "version": "1.0.0", # hopefully we get this from somewhere and not hardcode this + "name": model_name, + "description": "Automatically generated from Acumos model", + "component_type": "docker", + }, + "services": {"calls": [], "provides": []}, + "streams": {"subscribes": [], "publishes": []}, + "parameters": [], + "auxilary": {"healthcheck": {"type": "http", "endpoint": "/healthcheck"}}, + "artifacts": [{"type": "docker image", "uri": docker_uri}], + } + + # from https://pypi.org/project/acumos-dcae-model-runner/ + # each model method gets a subscruber and a publisher, using the methood name + pstype = "message_router" + for method in meta["methods"]: + + df_in_name = meta["methods"][method]["input"] + subscriber = { + "config_key": "{0}_subscriber".format(method), + "format": df_in_name, + "version": _get_format_version(df_in_name, data_formats), + "type": pstype, + } + + spec["streams"]["subscribes"].append(subscriber) + + df_out_name = meta["methods"][method]["output"] + + publisher = { + "config_key": "{0}_publisher".format(method), + "format": df_out_name, + "version": _get_format_version(df_out_name, data_formats), + "type": pstype, + } + + spec["streams"]["publishes"].append(publisher) + + # Validate that we have a valid spec + validate(instance=spec, schema=dcae_cs_schema) + + return spec + + +def generate_spec(model_repo_path, model_name, data_formats, docker_uri): + """ + Generate and write the component spec to disk + Returns the spec + """ + spec = _generate_spec( + model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri + ) + fname = "{0}_dcae_component_specification.json".format(model_name) + with open("{0}/{1}".format(model_repo_path, fname), "w") as f: + f.write(json.dumps(spec)) + + return spec diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py new file mode 100644 index 0000000..a5aae75 --- /dev/null +++ b/adapter/acumos/aoconversion/utils.py @@ -0,0 +1,24 @@ +# ============LICENSE_START==================================================== +# org.onap.dcae +# ============================================================================= +# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END====================================================== + +import json + + +def get_metadata(model_repo_path, model_name): + # for now, assume it's called "metadata.json" + return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read()) |