summaryrefslogtreecommitdiffstats
path: root/adapter/acumos/aoconversion
diff options
context:
space:
mode:
authorAndrew Gauld <agauld@att.com>2019-11-13 18:09:32 +0000
committerAndrew Gauld <agauld@att.com>2019-11-15 20:23:42 +0000
commit849da15d5b7ddc68e4c2b90b603fc8948d4b5e6d (patch)
treeb0d3d6e7dcd1551c007a7212dfdb369720707a22 /adapter/acumos/aoconversion
parent3f67c400813a60e4b8f9327e20eccc9033dc1b0b (diff)
Add acumos adapter project
Signed-off-by: Andrew Gauld <agauld@att.com> Issue-ID: DCAEGEN2-1860 Change-Id: Ib22fd2aa61fe7761bacf85e69540d11803c7acee Signed-off-by: Andrew Gauld <agauld@att.com>
Diffstat (limited to 'adapter/acumos/aoconversion')
-rw-r--r--adapter/acumos/aoconversion/__init__.py0
-rw-r--r--adapter/acumos/aoconversion/adapter.py50
-rw-r--r--adapter/acumos/aoconversion/convert.py33
-rw-r--r--adapter/acumos/aoconversion/dataformat_gen.py155
-rw-r--r--adapter/acumos/aoconversion/docker_gen.py110
-rw-r--r--adapter/acumos/aoconversion/exceptions.py29
-rw-r--r--adapter/acumos/aoconversion/index.html254
-rw-r--r--adapter/acumos/aoconversion/scanner.py312
-rw-r--r--adapter/acumos/aoconversion/spec_gen.py112
-rw-r--r--adapter/acumos/aoconversion/utils.py24
10 files changed, 1079 insertions, 0 deletions
diff --git a/adapter/acumos/aoconversion/__init__.py b/adapter/acumos/aoconversion/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/adapter/acumos/aoconversion/__init__.py
diff --git a/adapter/acumos/aoconversion/adapter.py b/adapter/acumos/aoconversion/adapter.py
new file mode 100644
index 0000000..38b5a71
--- /dev/null
+++ b/adapter/acumos/aoconversion/adapter.py
@@ -0,0 +1,50 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Command line for Acumos -> ONAP adapter
+"""
+
+import os
+import shutil
+import sys
+import yaml
+
+
+def adapter():
+ """
+ Run the adapter
+ """
+ import warnings
+ warnings.simplefilter('ignore')
+ with open(sys.argv[1], 'r') as f:
+ ycfg = yaml.safe_load(f.read())
+ from aoconversion import scanner
+ config = scanner.Config(**ycfg)
+ try:
+ shutil.rmtree(config.tmpdir)
+ except Exception:
+ pass
+ os.makedirs(config.tmpdir)
+ if config.port:
+ print('Starting web server')
+ scanner.serve(config)
+ else:
+ print('Starting scan')
+ scanner.scan(config)
+ print('Scan complete. Sleeping')
diff --git a/adapter/acumos/aoconversion/convert.py b/adapter/acumos/aoconversion/convert.py
new file mode 100644
index 0000000..e41820a
--- /dev/null
+++ b/adapter/acumos/aoconversion/convert.py
@@ -0,0 +1,33 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Module that converts acumos models to dcae artifacts
+"""
+from aoconversion import docker_gen, dataformat_gen, spec_gen
+
+
+def gen_dcae_artifacts_for_model(config, model_name, model_version="latest"):
+ """
+ Generate all dcae artifacts given an acumos model
+ """
+ model_repo_path = config.tmpdir
+ docker_uri = docker_gen.build_and_push_docker(config, model_name, model_version)
+ data_formats = dataformat_gen.generate_dcae_data_formats(model_repo_path, model_name)
+ spec = spec_gen.generate_spec(model_repo_path, model_name, data_formats, docker_uri)
+ return docker_uri, data_formats, spec
diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py
new file mode 100644
index 0000000..a5ead96
--- /dev/null
+++ b/adapter/acumos/aoconversion/dataformat_gen.py
@@ -0,0 +1,155 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+from subprocess import PIPE, Popen
+import json
+from jsonschema import validate
+import requests
+from aoconversion import utils, exceptions
+
+
+def _get_js_schema():
+ res = requests.get("http://json-schema.org/draft-04/schema#")
+ return res.json()
+
+
+def _get_dcae_df_schema():
+ res = requests.get(
+ "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD"
+ )
+ return res.json()
+
+
+def _protobuf_to_js(proto_path):
+ """
+ Converts a protobuf to jsonschema and returns the generated schema as a JSON object.
+ """
+ cmd = ["protobuf-jsonschema", proto_path]
+ p = Popen(cmd, stderr=PIPE, stdout=PIPE)
+ out = p.stdout.read()
+ asjson = json.loads(out)
+
+ # change the defintion names to remove the random package name that acumos generates
+ defs = asjson["definitions"]
+ defns = list(defs.keys())
+ for defn in defns:
+ # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key
+ defs[defn.split(".")[1]] = defs.pop(defn)
+
+ # make sure what we got out is a valid jsonschema
+ draft4 = _get_js_schema()
+ validate(instance=asjson, schema=draft4)
+
+ return asjson
+
+
+def _get_needed_formats(meta):
+ """
+ Read the metadata and figure out what the principle data formats are.
+ We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another.
+ We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format
+ """
+ # we use a dict because multiple methods may reuse names
+ needed_formats = {}
+ for method in meta["methods"]:
+ needed_formats[meta["methods"][method]["input"]] = 1
+ needed_formats[meta["methods"][method]["output"]] = 1
+ return list(needed_formats.keys())
+
+
+def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema):
+ """
+ Generates a collection of data formats from the model .proto
+ This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters,
+ but generate_dcae_data_formats requires some mocking etc.
+ """
+ js = _protobuf_to_js(proto_path)
+ needed_formats = _get_needed_formats(meta)
+
+ data_formats = []
+
+ used_defns = []
+
+ # iterate over and convert
+ for nf in needed_formats:
+ defn = js["definitions"][nf]
+
+ definitions = {}
+
+ # check for the case where we have an array of other defns
+ for prop in defn["properties"]:
+ if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]:
+ unclean_ref_name = defn["properties"][prop]["items"]["$ref"]
+ clean_ref_name = unclean_ref_name.split(".")[1]
+ if clean_ref_name in js["definitions"]:
+ defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name)
+ definitions[clean_ref_name] = js["definitions"][clean_ref_name]
+ used_defns.append(clean_ref_name)
+ else: # this is bad/unsupported, investigate
+ raise exceptions.UnsupportedFormatScenario()
+
+ # the defns created by this tool do not include a schema field.
+ # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12
+ defn["$schema"] = "http://json-schema.org/draft-04/schema#"
+
+ # Include the definitions, which may be empty {}
+ defn["definitions"] = definitions
+
+ # Validate that our resulting jsonschema is valid jsonschema
+ validate(instance=defn, schema=draft_4_schema)
+
+ # we currently hardcode dataformatversion, since it is the latest and has been for years https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD
+ dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn}
+
+ # make sure the schema validates against the DCAE data format schema
+ validate(instance=dcae_df, schema=dcae_df_schema)
+
+ # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats
+ data_formats.append(dcae_df)
+
+ # make sure every definitin we got out was used. Otherwise, this requires investigation!!
+ if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())):
+ raise exceptions.UnsupportedFormatScenario()
+
+ return data_formats
+
+
+# Public
+
+
+def generate_dcae_data_formats(model_repo_path, model_name):
+ """
+ Generates a collection of data formats from the model .proto
+ Writes them to disk
+ Returns them as the return of this call so this can be fed directly into spec gen
+ """
+ data_formats = _generate_dcae_data_formats(
+ "{0}/{1}/model.proto".format(model_repo_path, model_name),
+ utils.get_metadata(model_repo_path, model_name),
+ _get_dcae_df_schema(),
+ _get_js_schema(),
+ )
+
+ # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that
+ for df in data_formats:
+ # name_version seems like a reasonable filename
+ fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"])
+ with open("{0}/{1}".format(model_repo_path, fname), "w") as f:
+ f.write(json.dumps(df))
+
+ return data_formats
diff --git a/adapter/acumos/aoconversion/docker_gen.py b/adapter/acumos/aoconversion/docker_gen.py
new file mode 100644
index 0000000..bf54cfc
--- /dev/null
+++ b/adapter/acumos/aoconversion/docker_gen.py
@@ -0,0 +1,110 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import os
+from docker import APIClient
+from aoconversion import exceptions, utils
+
+
+def _generate_dockerfile(meta, model_name):
+ """
+ bind the templated docker string
+ """
+ docker_template = """
+ FROM python:{VERSION}
+
+ ENV MODELNAME {MODELNAME}
+ RUN mkdir /app
+ WORKDIR /app
+
+ ADD ./{MODELNAME} /app/{MODELNAME}
+ ADD ./requirements.txt /app
+
+ RUN pip install -r /app/requirements.txt && \
+ pip install acumos_dcae_model_runner
+
+ ENV DCAEPORT=10000
+ EXPOSE $DCAEPORT
+
+ ENTRYPOINT ["acumos_dcae_model_runner"]
+ CMD ["/app/{MODELNAME}"]
+ """
+ python_version = meta["runtime"]["version"]
+ return docker_template.format(VERSION=python_version, MODELNAME=model_name)
+
+
+# Public
+
+
+def build_and_push_docker(config, model_name, model_version="latest"):
+ """
+ build and push the dcae docker container
+ Returns the docker uri so this can be pipelined into specgen
+ """
+ model_repo_path = config.tmpdir
+ dockerfile_path = "{0}/Dockerfile".format(model_repo_path)
+ reqs_path = "{0}/requirements.txt".format(model_repo_path)
+
+ # get the metadata
+ meta = utils.get_metadata(model_repo_path, model_name)
+
+ # write the reqs file, will be removed later
+ reqs = meta["runtime"]["dependencies"]["pip"]["requirements"]
+ with open(reqs_path, "w") as f:
+ for r in reqs:
+ f.write("{0}=={1}\n".format(r["name"], r["version"]))
+
+ # generate the dockerfile
+ dockerfile = _generate_dockerfile(meta, model_name)
+
+ # write the dockerfile, will be removed later
+ with open("{0}/Dockerfile".format(model_repo_path), "w") as f:
+ f.write(dockerfile)
+
+ docker_uri = "{0}/{1}:{2}".format(config.dockerregistry, model_name, model_version)
+
+ # do the docker build
+ cli = APIClient(base_url=config.dockerhost, user_agent="Docker-Client-xx.yy")
+ response = [line.decode() for line in cli.build(path=model_repo_path, rm=True, tag=docker_uri)]
+
+ # clean up the files
+ os.remove(dockerfile_path)
+ os.remove(reqs_path)
+
+ # parse the Docker response to see whether we succeeded
+ for r in response:
+ # In some scenarios, for example a non-existing Dockerfile, docker build raises a native exception, see:
+ # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build
+ # However, if something fails such as a non-existing directory referenced in the dockerfile, NO exception is raised.
+ # In this case, one of the console output lines is "errorDetail"....
+ if "errorDetail" in r:
+ raise exceptions.DockerBuildFail(r)
+
+ # if the above succeeded, we can push
+ # push. same problem as above; e.g., "no basic auth credentials" does not throw an exception!
+ response = [
+ line.decode()
+ for line in cli.push(
+ repository=docker_uri, auth_config={"username": config.dockeruser, "password": config.dockerpass()}, stream=True
+ )
+ ]
+ for r in response:
+ if "errorDetail" in r:
+ raise exceptions.DockerPushFail(r)
+
+ return docker_uri
diff --git a/adapter/acumos/aoconversion/exceptions.py b/adapter/acumos/aoconversion/exceptions.py
new file mode 100644
index 0000000..8e87458
--- /dev/null
+++ b/adapter/acumos/aoconversion/exceptions.py
@@ -0,0 +1,29 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+
+class DockerBuildFail(BaseException):
+ pass
+
+
+class DockerPushFail(BaseException):
+ pass
+
+
+class UnsupportedFormatScenario(BaseException):
+ pass
diff --git a/adapter/acumos/aoconversion/index.html b/adapter/acumos/aoconversion/index.html
new file mode 100644
index 0000000..c94621a
--- /dev/null
+++ b/adapter/acumos/aoconversion/index.html
@@ -0,0 +1,254 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0//EN">
+<!--
+============LICENSE_START=======================================================
+org.onap.dcae
+================================================================================
+Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+================================================================================
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+============LICENSE_END=========================================================
+-->
+<html>
+<meta http-equiv="X-UA-Compatible" content="IE=Edge;chrome=1"/>
+<head><title>Acumos-ONAP Adapter Demo</title>
+<style>
+body {
+ background-size: cover;
+ font-family: Arial;
+ color: blue;
+ background-color: white;
+}
+ul {
+ margin: 0;
+ padding: 0;
+ list-style-type: none;
+}
+ul li {
+ float: left;
+ width: 200px;
+ height: 50px;
+ background-color: black;
+ opacity: 1;
+ line-height: 50px;
+ text-align: center;
+ font-size: 14px;
+ font-weight: 600;
+ text-transform: uppercase;
+}
+ul li a {
+ text-decoration: none;
+ color: white;
+ display: block;
+}
+ul li a:hover {
+ background-color: green;
+}
+ul li ul li {
+ display: none;
+}
+ul li:hover ul li {
+ display: block;
+}
+.navBar {
+ height: 50px;
+ background-color: black;
+}
+.auxNav {
+ float: right;
+}
+#itAcumos {
+ display: none;
+}
+#onboardingInProgress {
+ display: none;
+}
+.form-popup {
+ display: none;
+ position: fixed;
+ top: 100px;
+ left: 100px;
+ border: 3px solid #f1f1f1;
+ z-index: 9;
+}
+.form-container {
+ max-width: 350px;
+ padding: 10px;
+ background-color: white;
+}
+.form-container input[type=text], .form-container input[type=password] {
+ width: 300px;
+ padding: 15px;
+ margin: 5px 0 22px 0;
+ border: none;
+ background: #f1f1f1;
+}
+</style>
+</head><body>
+<div class="navBar">
+<ul>
+<li><a>File</a>
+ <ul>
+ <li><a onclick="openImportPopup()">Import ...</a></li>
+ </ul>
+</li>
+<li class="auxNav"><a>Signout</a></li>
+</ul>
+</div>
+<form class="form-popup form-container" id="importPopup">
+ <h2 align="center">Import Model</h2>
+ <select id="importType" onchange="setImportType()">
+ <option value="-" selected="true">-- Choose Import Type --</option>
+ <option value="itAcumos" >Acumos</option>
+ </select>
+ <div id="itAcumos">
+ <hr/>
+ <label for="furl"><b>Acumos Federation URL</b></label> <input id="furl" type="text" name="furl" placeholder="https://server:9084" required>
+ <button type="button" onclick="lookupCatalogs()">Lookup</button>
+ </div>
+ <div id="cAcumos">
+ <hr/>
+ <label for="catMenu"><b>Select Catalog</b></label> <select id="catMenu" onchange="chooseCatalog()">
+ <option value="*">All Catalogs</option>
+ </select>
+ </div>
+ <div id="acSols">
+ <hr/>
+ <label for="solMenu"><b>Select Solution</b></label> <select id="solMenu" onchange="chooseSolution()">
+ <option value="*">All Solutions</option>
+ </select>
+ </div>
+ <div id="acRevs">
+ <hr/>
+ <label for="revMenu"><b>Select Revision</b></label> <select id="revMenu">
+ <option value="*">All Revisions</option>
+ </select>
+ </div>
+ <hr/>
+ <button type="button" onclick="closeImportPopup()">Cancel</button>
+ <button id="onboard" type="button" onclick="onBoard()">Onboard</button>
+ <b id="onboardingInProgress">Onboarding - Please Wait ...</b>
+</form>
+<script>
+function fcomp(n) {
+ return document.getElementById(n);
+}
+function cvalue(n) {
+ return fcomp(n).value;
+}
+
+function uecvalue(n) {
+ return encodeURIComponent("" + cvalue(n));
+}
+
+function esc(s) {
+ return s.replace(/&/g,'&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function onBoard() {
+ fcomp("onboardingInProgress").style.display = "block";
+ var url = "/onboard.js?acumos=" + uecvalue("furl");
+ if (cvalue("catMenu") != "*") {
+ url += "&catalogId=" + uecvalue("catMenu");
+ if (cvalue("solMenu") != "*") {
+ url += "&solutionId=" + uecvalue("solMenu");
+ if (cvalue("revMenu") != "*") {
+ url += "&revisionId=" + uecvalue("revMenu");
+ }
+ }
+ }
+ let xhr = new XMLHttpRequest();
+ xhr.onerror = xhr.onload = function() {
+ fcomp("onboardingInProgress").style.display = "none";
+ }
+ xhr.open("POST", url);
+ xhr.send();
+}
+
+function chooseSolution() {
+ if (cvalue("solMenu") == "*") {
+ updatevis();
+ } else {
+ lookupItem("acRevs", "revMenu", "/listRevisions.js?acumos=" + uecvalue("furl") + "&solutionId=" + uecvalue("solMenu"));
+ }
+}
+function chooseCatalog() {
+ if (cvalue("catMenu") == "*") {
+ updatevis();
+ } else {
+ lookupItem("acSols", "solMenu", "/listSolutions.js?acumos=" + uecvalue("furl") + "&catalogId=" + uecvalue("catMenu"));
+ }
+}
+function lookupCatalogs() {
+ fcomp("onboard").style.display = "block";
+ lookupItem("cAcumos", "catMenu", "/listCatalogs.js?acumos=" + uecvalue("furl"));
+}
+function lookupItem(dblock, smenu, url) {
+ fcomp(dblock).style.display = "block";
+ let xhr = new XMLHttpRequest();
+ let catmenu = fcomp(smenu);
+ catmenu.options.length = 1;
+ catmenu.options[0].selected = true;
+ xhr.onload = function() {
+ let catresp = JSON.parse(this.response);
+ var i;
+ for (i = 0; i < catresp.length; i++) {
+ var option = document.createElement("option");
+ option.text = esc(catresp[i].name);
+ option.value = catresp[i].id;
+ catmenu.add(option);
+ }
+ updatevis();
+ };
+ xhr.open("GET", url);
+ xhr.send();
+}
+
+function updatevis() {
+ if (cvalue("importType") != "itAcumos") {
+ fcomp("itAcumos").style.display = "none";
+ fcomp("furl").value = "";
+ }
+ if (cvalue("furl") == "") {
+ fcomp("cAcumos").style.display = "none";
+ fcomp("onboard").style.display = "none";
+ fcomp("catMenu").options[0].selected = true;
+ }
+ if (cvalue("catMenu") == "*") {
+ fcomp("acSols").style.display = "none";
+ fcomp("solMenu").options[0].selected = true;
+ }
+ if (cvalue("solMenu") == "*") {
+ fcomp("acRevs").style.display = "none";
+ fcomp("revMenu").options[0].selected = true;
+ }
+}
+
+function setImportType() {
+ let di = fcomp("itAcumos");
+ if (cvalue("importType") == "itAcumos") {
+ fcomp("furl").value = "";
+ di.style.display = "block";
+ }
+ updatevis();
+}
+function openImportPopup() {
+ fcomp("importType").options[0].selected = true;
+ fcomp("importPopup").style.display = "block";
+ updatevis();
+}
+function closeImportPopup() {
+ fcomp("importPopup").style.display = "none";
+}
+</script>
+</body>
+</html>
diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py
new file mode 100644
index 0000000..22a5922
--- /dev/null
+++ b/adapter/acumos/aoconversion/scanner.py
@@ -0,0 +1,312 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import json
+import os
+from pkg_resources import resource_string
+import shutil
+import traceback
+try:
+ from urllib.parse import unquote_plus
+ from socketserver import ThreadingMixIn
+ from http.server import BaseHTTPRequestHandler, HTTPServer
+except ImportError:
+ from urllib import unquote_plus
+ from SocketServer import ThreadingMixIn
+ from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+import requests
+
+from aoconversion import convert
+
+
+def _derefconfig(value):
+ if value.startswith('@'):
+ with open(value[1:], 'r') as f:
+ return f.readline().strip()
+ return value
+
+
+class Config(object):
+ """
+ Configuration parameters as attributes, make sure the required ones are there,
+ populate defaults.
+ """
+ def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='unix:///var/run/docker.sock', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras):
+ self.dcaeurl = dcaeurl
+ self.dcaeuser = dcaeuser
+
+ def x(fmt, *args, **kwargs):
+ return onboardingurl + fmt.format(*args, **kwargs)
+ self.oburl = x
+ self._onboardingpass = onboardingpass
+ self._onboardinguser = onboardinguser
+ self.acumosurl = acumosurl
+ self.certfile = certfile
+ self.certverify = certverify
+ self.dockerhost = dockerhost
+ self.dockerregistry = dockerregistry
+ self.dockeruser = dockeruser
+ self._dockerpass = dockerpass
+ self.interval = interval
+ self.tmpdir = tmpdir
+ if catalogs is not None and type(catalogs) is not list:
+ catalogs = [catalogs]
+ self.catalogs = catalogs
+ self.port = port
+
+ def obauth(self):
+ return (self._onboardinguser, _derefconfig(self._onboardingpass))
+
+ def dockerpass(self):
+ return _derefconfig(self._dockerpass)
+
+
+class _AcumosAccess(object):
+ def __init__(self, config, url):
+ self.cert = config.certfile
+ self.verify = config.certverify
+ self.url = url.strip().rstrip('/')
+
+ def artgetter(self, xrev, matcher):
+ for art in xrev['artifacts']:
+ if matcher(art):
+ def ret():
+ nurl = '{}/artifacts/{}/content'.format(self.url, art['artifactId'])
+ resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify)
+ if resp.status_code == 500:
+ resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify)
+ resp.raise_for_status()
+ return resp
+ return ret
+ return None
+
+ def jsonget(self, format, *args, **kwargs):
+ nurl = self.url + format.format(*args, **kwargs)
+ resp = requests.get(nurl, cert=self.cert, verify=self.verify)
+ if resp.status_code == 500:
+ resp = requests.get(nurl, cert=self.cert, verify=self.verify)
+ resp.raise_for_status()
+ return resp.json()['content']
+
+
+def _x_proto_matcher(art):
+ """ Is this artifact the x.proto file? """
+ return art['name'].endswith('.proto')
+
+
+def _x_zip_matcher(art):
+ """ Is this artifact the x.zip file? """
+ return art['name'].endswith('.zip')
+
+
+def _md_json_matcher(art):
+ """ Is this artifact the metadata.json file? """
+ return art['name'].endswith('.json')
+
+
+def _walk(config):
+ """
+ Walk the Federation E5 interface of an Acumos instance
+ """
+ url = config.acumosurl
+ callback = _makecallback(config)
+ catalogs = config.catalogs
+ aa = _AcumosAccess(config, url)
+ for catalog in aa.jsonget('/catalogs'):
+ if catalogs is not None and catalog['catalogId'] not in catalogs and catalog['name'] not in catalogs:
+ continue
+ for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']):
+ for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+ onboard(aa, callback, solution, revision['revisionId'])
+
+
+def onboard(aa, callback, solution, revid):
+ xrev = aa.jsonget('/solutions/{}/revisions/{}', solution['solutionId'], revid)
+ callback(model_name=solution['name'], model_version=xrev['version'], model_last_updated=xrev['modified'], rating=solution['ratingAverageTenths'] / 10.0, proto_getter=aa.artgetter(xrev, _x_proto_matcher), zip_getter=aa.artgetter(xrev, _x_zip_matcher), metadata_getter=aa.artgetter(xrev, _md_json_matcher))
+
+
+def _pullfile(source, dest):
+ with open(dest, 'wb') as f:
+ for chunk in source().iter_content(65536):
+ f.write(chunk)
+
+
+_loadedformats = set()
+_loadedcomponents = set()
+
+
+def scan(config):
+ _walk(config)
+
+
+def _makecallback(config):
+ workdir = config.tmpdir
+ obauth = config.obauth()
+ oburl = config.oburl
+
+ def callback(model_name, model_version, model_last_updated, rating, proto_getter, zip_getter, metadata_getter):
+ model_name = model_name.lower()
+ model_version = model_version.lower()
+ compid = (model_name, model_version)
+ if compid in _loadedcomponents:
+ print('Skipping component {}: already analyzed'.format(compid))
+ return
+ if proto_getter is None or zip_getter is None or metadata_getter is None:
+ print('Skipping component {}: does not have required artifacts'.format(compid))
+ _loadedcomponents.add(compid)
+ return
+ modeldir = '{}/{}'.format(workdir, model_name)
+ shutil.rmtree(modeldir, True)
+ os.makedirs(modeldir)
+ try:
+ _pullfile(proto_getter, '{}/model.proto'.format(modeldir))
+ _pullfile(zip_getter, '{}/model.zip'.format(modeldir))
+ _pullfile(metadata_getter, '{}/metadata.json'.format(modeldir))
+ except Exception:
+ print('Skipping component {}: artifact access error'.format(compid))
+ _loadedcomponents.add(compid)
+ return
+ try:
+ docker_uri, data_formats, spec = convert.gen_dcae_artifacts_for_model(config, model_name, model_version)
+ shutil.rmtree(modeldir)
+ except Exception:
+ print('Error analyzing artifacts for {}'.format(compid))
+ traceback.print_exc()
+ return
+ for data_format in data_formats:
+ fmtid = (data_format['self']['name'], data_format['self']['version'])
+ if fmtid in _loadedformats:
+ print('Skipping data format {}: already analyzed'.format(fmtid))
+ continue
+ try:
+ resp = requests.post(oburl('/dataformats'), json={'owner': config.dcaeuser, 'spec': data_format}, auth=obauth)
+ if resp.status_code == 409:
+ print('Skipping data format {}: previously loaded'.format(fmtid))
+ _loadedformats.add(fmtid)
+ continue
+ resp.raise_for_status()
+ requests.patch(resp.json()['dataFormatUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status()
+ print('Loaded data format {}'.format(fmtid))
+ _loadedformats.add(fmtid)
+ except Exception:
+ print('Error loading data format {}'.format(fmtid))
+ traceback.print_exc()
+ raise
+ try:
+ resp = requests.post(oburl('/components'), json={'owner': config.dcaeuser, 'spec': spec}, auth=obauth)
+ if resp.status_code == 409:
+ print('Skipping component {}: previously loaded'.format(compid))
+ _loadedcomponents.add(compid)
+ return
+ resp.raise_for_status()
+ requests.patch(resp.json()['componentUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status()
+ print('Loaded component {}'.format(compid))
+ _loadedcomponents.add(compid)
+ except Exception:
+ print('Error loading component {}'.format(compid))
+ traceback.print_exc()
+ raise
+ return callback
+
+
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+ protocol_version = "HTTP/1.1"
+
+
+class Apihandler(BaseHTTPRequestHandler):
+ def doqp(self):
+ self.qparams = {}
+ if not self.path or '?' not in self.path:
+ return
+ self.path, qp = self.path.split('?', 1)
+ for x in qp.split('&'):
+ k, v = x.split('=', 1)
+ self.qparams[unquote_plus(k)] = unquote_plus(v)
+
+ def replyjson(self, body, ctype='application/json'):
+ self.replyraw(json.dumps(body).encode('utf-8'), ctype)
+
+ def replyraw(self, data, ctype):
+ self.send_response(200)
+ self.send_header('Content-Type', ctype)
+ self.send_header('Content-Length', len(data))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def do_GET(self):
+ self.doqp()
+ if self.path == '/' or self.path == '/index.html':
+ self.replyraw(self.server.index, 'text/html')
+ return
+ if 'acumos' not in self.qparams:
+ self.send_error(400)
+ return
+ aa = _AcumosAccess(self.server.config, self.qparams['acumos'])
+ if self.path == '/listCatalogs.js':
+ self.replyjson([{'name': x['name'], 'id': x['catalogId']} for x in aa.jsonget('/catalogs')])
+ return
+ if self.path == '/listSolutions.js':
+ if 'catalogId' not in self.qparams:
+ self.send_error(400)
+ return
+ self.replyjson([{'name': x['name'], 'id': x['solutionId']} for x in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId'])])
+ return
+ if self.path == '/listRevisions.js':
+ if 'solutionId' not in self.qparams:
+ self.send_error(400)
+ return
+ self.replyjson([{'name': x['version'], 'id': x['revisionId']} for x in aa.jsonget('/solutions/{}/revisions', self.qparams['solutionId'])])
+ return
+ self.send_error(404)
+
+ def do_POST(self):
+ self.doqp()
+ if self.path == '/onboard.js':
+ if 'acumos' not in self.qparams:
+ self.send_error(400)
+ return
+ aa = _AcumosAccess(self.server.config, self.qparams['acumos'])
+ callback = self.server.callback
+ if 'catalogId' not in self.qparams:
+ for catalog in aa.jsonget('/catalogs'):
+ for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']):
+ for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+ onboard(aa, callback, solution, revision['revisionId'])
+ elif 'solutionId' not in self.qparams:
+ for solution in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId']):
+ for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+ onboard(aa, callback, solution, revision['revisionId'])
+ elif 'revisionId' not in self.qparams:
+ solution = aa.jsonget('/solutions/{}', self.qparams['solutionId'])
+ for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+ onboard(aa, callback, solution, revision['revisionId'])
+ else:
+ solution = aa.jsonget('/solutions/{}', self.qparams['solutionId'])
+ onboard(aa, callback, solution, self.qparams['revisionId'])
+ self.replyraw('OK', 'text/plain')
+ return
+ self.send_error(400)
+
+
+def serve(config):
+ server = ThreadedHTTPServer(('', config.port), Apihandler)
+ server.config = config
+ server.callback = _makecallback(config)
+ server.index = resource_string(__name__, 'index.html')
+ server.serve_forever()
diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py
new file mode 100644
index 0000000..1662872
--- /dev/null
+++ b/adapter/acumos/aoconversion/spec_gen.py
@@ -0,0 +1,112 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Generates DCAE component specs
+"""
+
+
+import json
+from jsonschema import validate
+import requests
+from aoconversion import utils
+
+
+def _get_dcae_cs_schema():
+ res = requests.get(
+ "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD"
+ )
+ return res.json()
+
+
+def _get_format_version(target_name, data_formats):
+ """
+ search through the data formats for name, make sure we have it, and retrieve the version
+ """
+ # the df must exist, since the data formats were generated from the same metadata, or dataformats call blew up.
+ # So we don't do error checking here
+ for df in data_formats:
+ if df["self"]["name"] == target_name:
+ return df["self"]["version"]
+
+
+def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri):
+ """
+ Function that generates the component spec from the model metadata and docker info
+ Broken out to be unit-testable.
+ """
+
+ spec = {
+ "self": {
+ "version": "1.0.0", # hopefully we get this from somewhere and not hardcode this
+ "name": model_name,
+ "description": "Automatically generated from Acumos model",
+ "component_type": "docker",
+ },
+ "services": {"calls": [], "provides": []},
+ "streams": {"subscribes": [], "publishes": []},
+ "parameters": [],
+ "auxilary": {"healthcheck": {"type": "http", "endpoint": "/healthcheck"}},
+ "artifacts": [{"type": "docker image", "uri": docker_uri}],
+ }
+
+ # from https://pypi.org/project/acumos-dcae-model-runner/
+ # each model method gets a subscruber and a publisher, using the methood name
+ pstype = "message_router"
+ for method in meta["methods"]:
+
+ df_in_name = meta["methods"][method]["input"]
+ subscriber = {
+ "config_key": "{0}_subscriber".format(method),
+ "format": df_in_name,
+ "version": _get_format_version(df_in_name, data_formats),
+ "type": pstype,
+ }
+
+ spec["streams"]["subscribes"].append(subscriber)
+
+ df_out_name = meta["methods"][method]["output"]
+
+ publisher = {
+ "config_key": "{0}_publisher".format(method),
+ "format": df_out_name,
+ "version": _get_format_version(df_out_name, data_formats),
+ "type": pstype,
+ }
+
+ spec["streams"]["publishes"].append(publisher)
+
+ # Validate that we have a valid spec
+ validate(instance=spec, schema=dcae_cs_schema)
+
+ return spec
+
+
+def generate_spec(model_repo_path, model_name, data_formats, docker_uri):
+ """
+ Generate and write the component spec to disk
+ Returns the spec
+ """
+ spec = _generate_spec(
+ model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri
+ )
+ fname = "{0}_dcae_component_specification.json".format(model_name)
+ with open("{0}/{1}".format(model_repo_path, fname), "w") as f:
+ f.write(json.dumps(spec))
+
+ return spec
diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py
new file mode 100644
index 0000000..a5aae75
--- /dev/null
+++ b/adapter/acumos/aoconversion/utils.py
@@ -0,0 +1,24 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import json
+
+
+def get_metadata(model_repo_path, model_name):
+ # for now, assume it's called "metadata.json"
+ return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read())