Add acumos adapter project

Signed-off-by: Andrew Gauld <agauld@att.com>
Issue-ID: DCAEGEN2-1860
Change-Id: Ib22fd2aa61fe7761bacf85e69540d11803c7acee
Signed-off-by: Andrew Gauld <agauld@att.com>

10 files changed, 1079 insertions, 0 deletions

diff --git a/adapter/acumos/aoconversion/__init__.py b/adapter/acumos/aoconversion/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/adapter/acumos/aoconversion/__init__.py
diff --git a/adapter/acumos/aoconversion/adapter.py b/adapter/acumos/aoconversion/adapter.py
new file mode 100644
index 0000000..38b5a71
--- /dev/null
+++ b/adapter/acumos/aoconversion/adapter.py
@@ -0,0 +1,50 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Command line for Acumos -> ONAP adapter
+"""
+
+import os
+import shutil
+import sys
+import yaml
+
+
+def adapter():
+    """
+    Run the adapter
+    """
+    import warnings
+    warnings.simplefilter('ignore')
+    with open(sys.argv[1], 'r') as f:
+        ycfg = yaml.safe_load(f.read())
+    from aoconversion import scanner
+    config = scanner.Config(**ycfg)
+    try:
+        shutil.rmtree(config.tmpdir)
+    except Exception:
+        pass
+    os.makedirs(config.tmpdir)
+    if config.port:
+        print('Starting web server')
+        scanner.serve(config)
+    else:
+        print('Starting scan')
+        scanner.scan(config)
+        print('Scan complete.    Sleeping')
diff --git a/adapter/acumos/aoconversion/convert.py b/adapter/acumos/aoconversion/convert.py
new file mode 100644
index 0000000..e41820a
--- /dev/null
+++ b/adapter/acumos/aoconversion/convert.py
@@ -0,0 +1,33 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Module that converts acumos models to dcae artifacts
+"""
+from aoconversion import docker_gen, dataformat_gen, spec_gen
+
+
+def gen_dcae_artifacts_for_model(config, model_name, model_version="latest"):
+    """
+    Generate all dcae artifacts given an acumos model
+    """
+    model_repo_path = config.tmpdir
+    docker_uri = docker_gen.build_and_push_docker(config, model_name, model_version)
+    data_formats = dataformat_gen.generate_dcae_data_formats(model_repo_path, model_name)
+    spec = spec_gen.generate_spec(model_repo_path, model_name, data_formats, docker_uri)
+    return docker_uri, data_formats, spec
diff --git a/adapter/acumos/aoconversion/dataformat_gen.py b/adapter/acumos/aoconversion/dataformat_gen.py
new file mode 100644
index 0000000..a5ead96
--- /dev/null
+++ b/adapter/acumos/aoconversion/dataformat_gen.py
@@ -0,0 +1,155 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+from subprocess import PIPE, Popen
+import json
+from jsonschema import validate
+import requests
+from aoconversion import utils, exceptions
+
+
+def _get_js_schema():
+    res = requests.get("http://json-schema.org/draft-04/schema#")
+    return res.json()
+
+
+def _get_dcae_df_schema():
+    res = requests.get(
+        "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD"
+    )
+    return res.json()
+
+
+def _protobuf_to_js(proto_path):
+    """
+    Converts a protobuf to jsonschema and returns the generated schema as a JSON object.
+    """
+    cmd = ["protobuf-jsonschema", proto_path]
+    p = Popen(cmd, stderr=PIPE, stdout=PIPE)
+    out = p.stdout.read()
+    asjson = json.loads(out)
+
+    # change the defintion names to remove the random package name that acumos generates
+    defs = asjson["definitions"]
+    defns = list(defs.keys())
+    for defn in defns:
+        # https://stackoverflow.com/questions/16475384/rename-a-dictionary-key
+        defs[defn.split(".")[1]] = defs.pop(defn)
+
+    # make sure what we got out is a valid jsonschema
+    draft4 = _get_js_schema()
+    validate(instance=asjson, schema=draft4)
+
+    return asjson
+
+
+def _get_needed_formats(meta):
+    """
+    Read the metadata and figure out what the principle data formats are.
+    We cannot determine this from the proto because the proto may list "submessages" in a flat namespace; some of them may not coorespond to a data format but rather a referenced defintion in another.
+    We don't want to generate a data format for submessages though; instead they should be included in definitions as part of the relevent data format
+    """
+    # we use a dict because multiple methods may reuse names
+    needed_formats = {}
+    for method in meta["methods"]:
+        needed_formats[meta["methods"][method]["input"]] = 1
+        needed_formats[meta["methods"][method]["output"]] = 1
+    return list(needed_formats.keys())
+
+
+def _generate_dcae_data_formats(proto_path, meta, dcae_df_schema, draft_4_schema):
+    """
+    Generates a collection of data formats from the model .proto
+    This helper function is broken out for the ease of unit testing; this can be unit tested easily because all deps are parameters,
+    but generate_dcae_data_formats requires some mocking etc.
+    """
+    js = _protobuf_to_js(proto_path)
+    needed_formats = _get_needed_formats(meta)
+
+    data_formats = []
+
+    used_defns = []
+
+    # iterate over and convert
+    for nf in needed_formats:
+        defn = js["definitions"][nf]
+
+        definitions = {}
+
+        # check for the case where we have an array of other defns
+        for prop in defn["properties"]:
+            if defn["properties"][prop]["type"] == "array" and "$ref" in defn["properties"][prop]["items"]:
+                unclean_ref_name = defn["properties"][prop]["items"]["$ref"]
+                clean_ref_name = unclean_ref_name.split(".")[1]
+                if clean_ref_name in js["definitions"]:
+                    defn["properties"][prop]["items"]["$ref"] = "#/definitions/{0}".format(clean_ref_name)
+                    definitions[clean_ref_name] = js["definitions"][clean_ref_name]
+                    used_defns.append(clean_ref_name)
+                else:  # this is bad/unsupported, investigate
+                    raise exceptions.UnsupportedFormatScenario()
+
+        # the defns created by this tool do not include a schema field.
+        # I created an issue: https://github.com/devongovett/protobuf-jsonschema/issues/12
+        defn["$schema"] = "http://json-schema.org/draft-04/schema#"
+
+        # Include the definitions, which may be empty {}
+        defn["definitions"] = definitions
+
+        # Validate that our resulting jsonschema is valid jsonschema
+        validate(instance=defn, schema=draft_4_schema)
+
+        # we currently hardcode dataformatversion, since it is the latest and has been for years  https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/data-format/dcae-cli-v1/data-format-schema.json;hb=HEAD
+        dcae_df = {"self": {"name": nf, "version": "1.0.0"}, "dataformatversion": "1.0.1", "jsonschema": defn}
+
+        # make sure the schema validates against the DCAE data format schema
+        validate(instance=dcae_df, schema=dcae_df_schema)
+
+        # if we've passed the validation and exc raising so far, we are good, append this to output list of dcae data formats
+        data_formats.append(dcae_df)
+
+    # make sure every definitin we got out was used. Otherwise, this requires investigation!!
+    if sorted(needed_formats + used_defns) != sorted(list(js["definitions"].keys())):
+        raise exceptions.UnsupportedFormatScenario()
+
+    return data_formats
+
+
+# Public
+
+
+def generate_dcae_data_formats(model_repo_path, model_name):
+    """
+    Generates a collection of data formats from the model .proto
+    Writes them to disk
+    Returns them as the return of this call so this can be fed directly into spec gen
+    """
+    data_formats = _generate_dcae_data_formats(
+        "{0}/{1}/model.proto".format(model_repo_path, model_name),
+        utils.get_metadata(model_repo_path, model_name),
+        _get_dcae_df_schema(),
+        _get_js_schema(),
+    )
+
+    # now we iterate over these and write a file to disk for each, since the dcae cli seems to want that
+    for df in data_formats:
+        # name_version seems like a reasonable filename
+        fname = "{0}_{1}_dcae_data_format.json".format(df["self"]["name"], df["self"]["version"])
+        with open("{0}/{1}".format(model_repo_path, fname), "w") as f:
+            f.write(json.dumps(df))
+
+    return data_formats
diff --git a/adapter/acumos/aoconversion/docker_gen.py b/adapter/acumos/aoconversion/docker_gen.py
new file mode 100644
index 0000000..bf54cfc
--- /dev/null
+++ b/adapter/acumos/aoconversion/docker_gen.py
@@ -0,0 +1,110 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import os
+from docker import APIClient
+from aoconversion import exceptions, utils
+
+
+def _generate_dockerfile(meta, model_name):
+    """
+    bind the templated docker string
+    """
+    docker_template = """
+    FROM python:{VERSION}
+
+    ENV MODELNAME {MODELNAME}
+    RUN mkdir /app
+    WORKDIR /app
+
+    ADD ./{MODELNAME} /app/{MODELNAME}
+    ADD ./requirements.txt /app
+
+    RUN pip install -r /app/requirements.txt && \
+        pip install acumos_dcae_model_runner
+
+    ENV DCAEPORT=10000
+    EXPOSE $DCAEPORT
+
+    ENTRYPOINT ["acumos_dcae_model_runner"]
+    CMD ["/app/{MODELNAME}"]
+    """
+    python_version = meta["runtime"]["version"]
+    return docker_template.format(VERSION=python_version, MODELNAME=model_name)
+
+
+# Public
+
+
+def build_and_push_docker(config, model_name, model_version="latest"):
+    """
+    build and push the dcae docker container
+    Returns the docker uri so this can be pipelined into specgen
+    """
+    model_repo_path = config.tmpdir
+    dockerfile_path = "{0}/Dockerfile".format(model_repo_path)
+    reqs_path = "{0}/requirements.txt".format(model_repo_path)
+
+    # get the metadata
+    meta = utils.get_metadata(model_repo_path, model_name)
+
+    # write the reqs file, will be removed later
+    reqs = meta["runtime"]["dependencies"]["pip"]["requirements"]
+    with open(reqs_path, "w") as f:
+        for r in reqs:
+            f.write("{0}=={1}\n".format(r["name"], r["version"]))
+
+    # generate the dockerfile
+    dockerfile = _generate_dockerfile(meta, model_name)
+
+    # write the dockerfile, will be removed later
+    with open("{0}/Dockerfile".format(model_repo_path), "w") as f:
+        f.write(dockerfile)
+
+    docker_uri = "{0}/{1}:{2}".format(config.dockerregistry, model_name, model_version)
+
+    # do the docker build
+    cli = APIClient(base_url=config.dockerhost, user_agent="Docker-Client-xx.yy")
+    response = [line.decode() for line in cli.build(path=model_repo_path, rm=True, tag=docker_uri)]
+
+    # clean up the files
+    os.remove(dockerfile_path)
+    os.remove(reqs_path)
+
+    # parse the Docker response to see whether we succeeded
+    for r in response:
+        # In some scenarios, for example a non-existing Dockerfile, docker build raises a native exception, see:
+        # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build
+        # However, if something fails such as a non-existing directory referenced in the dockerfile, NO exception is raised.
+        # In this case, one of the console output lines is "errorDetail"....
+        if "errorDetail" in r:
+            raise exceptions.DockerBuildFail(r)
+
+    # if the above succeeded, we can push
+    # push. same problem as above; e.g., "no basic auth credentials" does not throw an exception!
+    response = [
+        line.decode()
+        for line in cli.push(
+            repository=docker_uri, auth_config={"username": config.dockeruser, "password": config.dockerpass()}, stream=True
+        )
+    ]
+    for r in response:
+        if "errorDetail" in r:
+            raise exceptions.DockerPushFail(r)
+
+    return docker_uri
diff --git a/adapter/acumos/aoconversion/exceptions.py b/adapter/acumos/aoconversion/exceptions.py
new file mode 100644
index 0000000..8e87458
--- /dev/null
+++ b/adapter/acumos/aoconversion/exceptions.py
@@ -0,0 +1,29 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+
+class DockerBuildFail(BaseException):
+    pass
+
+
+class DockerPushFail(BaseException):
+    pass
+
+
+class UnsupportedFormatScenario(BaseException):
+    pass
diff --git a/adapter/acumos/aoconversion/index.html b/adapter/acumos/aoconversion/index.html
new file mode 100644
index 0000000..c94621a
--- /dev/null
+++ b/adapter/acumos/aoconversion/index.html
@@ -0,0 +1,254 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0//EN">
+<!--
+============LICENSE_START=======================================================
+org.onap.dcae
+================================================================================
+Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+================================================================================
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+============LICENSE_END=========================================================
+-->
+<html>
+<meta http-equiv="X-UA-Compatible" content="IE=Edge;chrome=1"/>
+<head><title>Acumos-ONAP Adapter Demo</title>
+<style>
+body {
+  background-size: cover;
+  font-family: Arial;
+  color: blue;
+  background-color: white;
+}
+ul {
+  margin: 0;
+  padding: 0;
+  list-style-type: none;
+}
+ul li {
+  float: left;
+  width: 200px;
+  height: 50px;
+  background-color: black;
+  opacity: 1;
+  line-height: 50px;
+  text-align: center;
+  font-size: 14px;
+  font-weight: 600;
+  text-transform: uppercase;
+}
+ul li a {
+  text-decoration: none;
+  color: white;
+  display: block;
+}
+ul li a:hover {
+  background-color: green;
+}
+ul li ul li {
+  display: none;
+}
+ul li:hover ul li {
+  display: block;
+}
+.navBar {
+  height: 50px;
+  background-color: black;
+}
+.auxNav {
+  float: right;
+}
+#itAcumos {
+  display: none;
+}
+#onboardingInProgress {
+  display: none;
+}
+.form-popup {
+  display: none;
+  position: fixed;
+  top: 100px;
+  left: 100px;
+  border: 3px solid #f1f1f1;
+  z-index: 9;
+}
+.form-container {
+  max-width: 350px;
+  padding: 10px;
+  background-color: white;
+}
+.form-container input[type=text], .form-container input[type=password] {
+  width: 300px;
+  padding: 15px;
+  margin: 5px 0 22px 0;
+  border: none;
+  background: #f1f1f1;
+}
+</style>
+</head><body>
+<div class="navBar">
+<ul>
+<li><a>File</a>
+  <ul>
+    <li><a onclick="openImportPopup()">Import ...</a></li>
+  </ul>
+</li>
+<li class="auxNav"><a>Signout</a></li>
+</ul>
+</div>
+<form class="form-popup form-container" id="importPopup">
+  <h2 align="center">Import Model</h2>
+  <select id="importType" onchange="setImportType()">
+    <option value="-" selected="true">-- Choose Import Type --</option>
+    <option value="itAcumos" >Acumos</option>
+  </select>
+  <div id="itAcumos">
+    <hr/>
+    <label for="furl"><b>Acumos Federation URL</b></label> <input id="furl" type="text" name="furl" placeholder="https://server:9084" required>
+    <button type="button" onclick="lookupCatalogs()">Lookup</button>
+  </div>
+  <div id="cAcumos">
+    <hr/>
+    <label for="catMenu"><b>Select Catalog</b></label> <select id="catMenu" onchange="chooseCatalog()">
+      <option value="*">All Catalogs</option>
+    </select>
+  </div>
+  <div id="acSols">
+    <hr/>
+    <label for="solMenu"><b>Select Solution</b></label> <select id="solMenu" onchange="chooseSolution()">
+      <option value="*">All Solutions</option>
+    </select>
+  </div>
+  <div id="acRevs">
+    <hr/>
+    <label for="revMenu"><b>Select Revision</b></label> <select id="revMenu">
+      <option value="*">All Revisions</option>
+    </select>
+  </div>
+  <hr/>
+  <button type="button" onclick="closeImportPopup()">Cancel</button>
+  <button id="onboard" type="button" onclick="onBoard()">Onboard</button>
+  <b id="onboardingInProgress">Onboarding - Please Wait ...</b>
+</form>
+<script>
+function fcomp(n) {
+  return document.getElementById(n);
+}
+function cvalue(n) {
+  return fcomp(n).value;
+}
+
+function uecvalue(n) {
+  return encodeURIComponent("" + cvalue(n));
+}
+
+function esc(s) {
+  return s.replace(/&/g,'&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+function onBoard() {
+  fcomp("onboardingInProgress").style.display = "block";
+  var url = "/onboard.js?acumos=" + uecvalue("furl");
+  if (cvalue("catMenu") != "*") {
+    url += "&catalogId=" + uecvalue("catMenu");
+    if (cvalue("solMenu") != "*") {
+      url += "&solutionId=" + uecvalue("solMenu");
+      if (cvalue("revMenu") != "*") {
+        url += "&revisionId=" + uecvalue("revMenu");
+      }
+    }
+  }
+  let xhr = new XMLHttpRequest();
+  xhr.onerror = xhr.onload = function() {
+    fcomp("onboardingInProgress").style.display = "none";
+  }
+  xhr.open("POST", url);
+  xhr.send();
+}
+
+function chooseSolution() {
+  if (cvalue("solMenu") == "*") {
+    updatevis();
+  } else {
+    lookupItem("acRevs", "revMenu", "/listRevisions.js?acumos=" + uecvalue("furl") + "&solutionId=" + uecvalue("solMenu"));
+  }
+}
+function chooseCatalog() {
+  if (cvalue("catMenu") == "*") {
+    updatevis();
+  } else {
+    lookupItem("acSols", "solMenu", "/listSolutions.js?acumos=" + uecvalue("furl") + "&catalogId=" + uecvalue("catMenu"));
+  }
+}
+function lookupCatalogs() {
+  fcomp("onboard").style.display = "block";
+  lookupItem("cAcumos", "catMenu", "/listCatalogs.js?acumos=" + uecvalue("furl"));
+}
+function lookupItem(dblock, smenu, url) {
+  fcomp(dblock).style.display = "block";
+  let xhr = new XMLHttpRequest();
+  let catmenu = fcomp(smenu);
+  catmenu.options.length = 1;
+  catmenu.options[0].selected = true;
+  xhr.onload = function() {
+    let catresp = JSON.parse(this.response);
+    var i;
+    for (i = 0; i < catresp.length; i++) {
+      var option = document.createElement("option");
+      option.text = esc(catresp[i].name);
+      option.value = catresp[i].id;
+      catmenu.add(option);
+    }
+    updatevis();
+  };
+  xhr.open("GET", url);
+  xhr.send();
+}
+
+function updatevis() {
+  if (cvalue("importType") != "itAcumos") {
+    fcomp("itAcumos").style.display = "none";
+    fcomp("furl").value = "";
+  }
+  if (cvalue("furl") == "") {
+    fcomp("cAcumos").style.display = "none";
+    fcomp("onboard").style.display = "none";
+    fcomp("catMenu").options[0].selected = true;
+  }
+  if (cvalue("catMenu") == "*") {
+    fcomp("acSols").style.display = "none";
+    fcomp("solMenu").options[0].selected = true;
+  }
+  if (cvalue("solMenu") == "*") {
+    fcomp("acRevs").style.display = "none";
+    fcomp("revMenu").options[0].selected = true;
+  }
+}
+
+function setImportType() {
+  let di = fcomp("itAcumos");
+  if (cvalue("importType") == "itAcumos") {
+    fcomp("furl").value = "";
+    di.style.display = "block";
+  }
+  updatevis();
+}
+function openImportPopup() {
+  fcomp("importType").options[0].selected = true;
+  fcomp("importPopup").style.display = "block";
+  updatevis();
+}
+function closeImportPopup() {
+  fcomp("importPopup").style.display = "none";
+}
+</script>
+</body>
+</html>
diff --git a/adapter/acumos/aoconversion/scanner.py b/adapter/acumos/aoconversion/scanner.py
new file mode 100644
index 0000000..22a5922
--- /dev/null
+++ b/adapter/acumos/aoconversion/scanner.py
@@ -0,0 +1,312 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import json
+import os
+from pkg_resources import resource_string
+import shutil
+import traceback
+try:
+    from urllib.parse import unquote_plus
+    from socketserver import ThreadingMixIn
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+except ImportError:
+    from urllib import unquote_plus
+    from SocketServer import ThreadingMixIn
+    from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+import requests
+
+from aoconversion import convert
+
+
+def _derefconfig(value):
+    if value.startswith('@'):
+        with open(value[1:], 'r') as f:
+            return f.readline().strip()
+    return value
+
+
+class Config(object):
+    """
+    Configuration parameters as attributes, make sure the required ones are there,
+    populate defaults.
+    """
+    def __init__(self, dcaeurl, dcaeuser, onboardingurl, onboardinguser, onboardingpass, certfile, dockerregistry, dockeruser, dockerpass, acumosurl=None, interval=900, dockerhost='unix:///var/run/docker.sock', tmpdir='/var/tmp/aoadapter', certverify=True, catalogs=None, port=None, **extras):
+        self.dcaeurl = dcaeurl
+        self.dcaeuser = dcaeuser
+
+        def x(fmt, *args, **kwargs):
+            return onboardingurl + fmt.format(*args, **kwargs)
+        self.oburl = x
+        self._onboardingpass = onboardingpass
+        self._onboardinguser = onboardinguser
+        self.acumosurl = acumosurl
+        self.certfile = certfile
+        self.certverify = certverify
+        self.dockerhost = dockerhost
+        self.dockerregistry = dockerregistry
+        self.dockeruser = dockeruser
+        self._dockerpass = dockerpass
+        self.interval = interval
+        self.tmpdir = tmpdir
+        if catalogs is not None and type(catalogs) is not list:
+            catalogs = [catalogs]
+        self.catalogs = catalogs
+        self.port = port
+
+    def obauth(self):
+        return (self._onboardinguser, _derefconfig(self._onboardingpass))
+
+    def dockerpass(self):
+        return _derefconfig(self._dockerpass)
+
+
+class _AcumosAccess(object):
+    def __init__(self, config, url):
+        self.cert = config.certfile
+        self.verify = config.certverify
+        self.url = url.strip().rstrip('/')
+
+    def artgetter(self, xrev, matcher):
+        for art in xrev['artifacts']:
+            if matcher(art):
+                def ret():
+                    nurl = '{}/artifacts/{}/content'.format(self.url, art['artifactId'])
+                    resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify)
+                    if resp.status_code == 500:
+                        resp = requests.get(nurl, stream=True, cert=self.cert, verify=self.verify)
+                    resp.raise_for_status()
+                    return resp
+                return ret
+        return None
+
+    def jsonget(self, format, *args, **kwargs):
+        nurl = self.url + format.format(*args, **kwargs)
+        resp = requests.get(nurl, cert=self.cert, verify=self.verify)
+        if resp.status_code == 500:
+            resp = requests.get(nurl, cert=self.cert, verify=self.verify)
+        resp.raise_for_status()
+        return resp.json()['content']
+
+
+def _x_proto_matcher(art):
+    """ Is this artifact the x.proto file? """
+    return art['name'].endswith('.proto')
+
+
+def _x_zip_matcher(art):
+    """ Is this artifact the x.zip file? """
+    return art['name'].endswith('.zip')
+
+
+def _md_json_matcher(art):
+    """ Is this artifact the metadata.json file? """
+    return art['name'].endswith('.json')
+
+
+def _walk(config):
+    """
+    Walk the Federation E5 interface of an Acumos instance
+    """
+    url = config.acumosurl
+    callback = _makecallback(config)
+    catalogs = config.catalogs
+    aa = _AcumosAccess(config, url)
+    for catalog in aa.jsonget('/catalogs'):
+        if catalogs is not None and catalog['catalogId'] not in catalogs and catalog['name'] not in catalogs:
+            continue
+        for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']):
+            for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+                onboard(aa, callback, solution, revision['revisionId'])
+
+
+def onboard(aa, callback, solution, revid):
+    xrev = aa.jsonget('/solutions/{}/revisions/{}', solution['solutionId'], revid)
+    callback(model_name=solution['name'], model_version=xrev['version'], model_last_updated=xrev['modified'], rating=solution['ratingAverageTenths'] / 10.0, proto_getter=aa.artgetter(xrev, _x_proto_matcher), zip_getter=aa.artgetter(xrev, _x_zip_matcher), metadata_getter=aa.artgetter(xrev, _md_json_matcher))
+
+
+def _pullfile(source, dest):
+    with open(dest, 'wb') as f:
+        for chunk in source().iter_content(65536):
+            f.write(chunk)
+
+
+_loadedformats = set()
+_loadedcomponents = set()
+
+
+def scan(config):
+    _walk(config)
+
+
+def _makecallback(config):
+    workdir = config.tmpdir
+    obauth = config.obauth()
+    oburl = config.oburl
+
+    def callback(model_name, model_version, model_last_updated, rating, proto_getter, zip_getter, metadata_getter):
+        model_name = model_name.lower()
+        model_version = model_version.lower()
+        compid = (model_name, model_version)
+        if compid in _loadedcomponents:
+            print('Skipping component {}: already analyzed'.format(compid))
+            return
+        if proto_getter is None or zip_getter is None or metadata_getter is None:
+            print('Skipping component {}: does not have required artifacts'.format(compid))
+            _loadedcomponents.add(compid)
+            return
+        modeldir = '{}/{}'.format(workdir, model_name)
+        shutil.rmtree(modeldir, True)
+        os.makedirs(modeldir)
+        try:
+            _pullfile(proto_getter, '{}/model.proto'.format(modeldir))
+            _pullfile(zip_getter, '{}/model.zip'.format(modeldir))
+            _pullfile(metadata_getter, '{}/metadata.json'.format(modeldir))
+        except Exception:
+            print('Skipping component {}: artifact access error'.format(compid))
+            _loadedcomponents.add(compid)
+            return
+        try:
+            docker_uri, data_formats, spec = convert.gen_dcae_artifacts_for_model(config, model_name, model_version)
+            shutil.rmtree(modeldir)
+        except Exception:
+            print('Error analyzing artifacts for {}'.format(compid))
+            traceback.print_exc()
+            return
+        for data_format in data_formats:
+            fmtid = (data_format['self']['name'], data_format['self']['version'])
+            if fmtid in _loadedformats:
+                print('Skipping data format {}: already analyzed'.format(fmtid))
+                continue
+            try:
+                resp = requests.post(oburl('/dataformats'), json={'owner': config.dcaeuser, 'spec': data_format}, auth=obauth)
+                if resp.status_code == 409:
+                    print('Skipping data format {}: previously loaded'.format(fmtid))
+                    _loadedformats.add(fmtid)
+                    continue
+                resp.raise_for_status()
+                requests.patch(resp.json()['dataFormatUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status()
+                print('Loaded data format {}'.format(fmtid))
+                _loadedformats.add(fmtid)
+            except Exception:
+                print('Error loading data format {}'.format(fmtid))
+                traceback.print_exc()
+                raise
+        try:
+            resp = requests.post(oburl('/components'), json={'owner': config.dcaeuser, 'spec': spec}, auth=obauth)
+            if resp.status_code == 409:
+                print('Skipping component {}: previously loaded'.format(compid))
+                _loadedcomponents.add(compid)
+                return
+            resp.raise_for_status()
+            requests.patch(resp.json()['componentUrl'], json={'owner': config.dcaeuser, 'status': 'published'}, auth=obauth).raise_for_status()
+            print('Loaded component {}'.format(compid))
+            _loadedcomponents.add(compid)
+        except Exception:
+            print('Error loading component {}'.format(compid))
+            traceback.print_exc()
+            raise
+    return callback
+
+
+class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
+    protocol_version = "HTTP/1.1"
+
+
+class Apihandler(BaseHTTPRequestHandler):
+    def doqp(self):
+        self.qparams = {}
+        if not self.path or '?' not in self.path:
+            return
+        self.path, qp = self.path.split('?', 1)
+        for x in qp.split('&'):
+            k, v = x.split('=', 1)
+            self.qparams[unquote_plus(k)] = unquote_plus(v)
+
+    def replyjson(self, body, ctype='application/json'):
+        self.replyraw(json.dumps(body).encode('utf-8'), ctype)
+
+    def replyraw(self, data, ctype):
+        self.send_response(200)
+        self.send_header('Content-Type', ctype)
+        self.send_header('Content-Length', len(data))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self):
+        self.doqp()
+        if self.path == '/' or self.path == '/index.html':
+            self.replyraw(self.server.index, 'text/html')
+            return
+        if 'acumos' not in self.qparams:
+            self.send_error(400)
+            return
+        aa = _AcumosAccess(self.server.config, self.qparams['acumos'])
+        if self.path == '/listCatalogs.js':
+            self.replyjson([{'name': x['name'], 'id': x['catalogId']} for x in aa.jsonget('/catalogs')])
+            return
+        if self.path == '/listSolutions.js':
+            if 'catalogId' not in self.qparams:
+                self.send_error(400)
+                return
+            self.replyjson([{'name': x['name'], 'id': x['solutionId']} for x in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId'])])
+            return
+        if self.path == '/listRevisions.js':
+            if 'solutionId' not in self.qparams:
+                self.send_error(400)
+                return
+            self.replyjson([{'name': x['version'], 'id': x['revisionId']} for x in aa.jsonget('/solutions/{}/revisions', self.qparams['solutionId'])])
+            return
+        self.send_error(404)
+
+    def do_POST(self):
+        self.doqp()
+        if self.path == '/onboard.js':
+            if 'acumos' not in self.qparams:
+                self.send_error(400)
+                return
+            aa = _AcumosAccess(self.server.config, self.qparams['acumos'])
+            callback = self.server.callback
+            if 'catalogId' not in self.qparams:
+                for catalog in aa.jsonget('/catalogs'):
+                    for solution in aa.jsonget('/solutions?catalogId={}', catalog['catalogId']):
+                        for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+                            onboard(aa, callback, solution, revision['revisionId'])
+            elif 'solutionId' not in self.qparams:
+                for solution in aa.jsonget('/solutions?catalogId={}', self.qparams['catalogId']):
+                    for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+                        onboard(aa, callback, solution, revision['revisionId'])
+            elif 'revisionId' not in self.qparams:
+                solution = aa.jsonget('/solutions/{}', self.qparams['solutionId'])
+                for revision in aa.jsonget('/solutions/{}/revisions', solution['solutionId']):
+                    onboard(aa, callback, solution, revision['revisionId'])
+            else:
+                solution = aa.jsonget('/solutions/{}', self.qparams['solutionId'])
+                onboard(aa, callback, solution, self.qparams['revisionId'])
+            self.replyraw('OK', 'text/plain')
+            return
+        self.send_error(400)
+
+
+def serve(config):
+    server = ThreadedHTTPServer(('', config.port), Apihandler)
+    server.config = config
+    server.callback = _makecallback(config)
+    server.index = resource_string(__name__, 'index.html')
+    server.serve_forever()
diff --git a/adapter/acumos/aoconversion/spec_gen.py b/adapter/acumos/aoconversion/spec_gen.py
new file mode 100644
index 0000000..1662872
--- /dev/null
+++ b/adapter/acumos/aoconversion/spec_gen.py
@@ -0,0 +1,112 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+"""
+Generates DCAE component specs
+"""
+
+
+import json
+from jsonschema import validate
+import requests
+from aoconversion import utils
+
+
+def _get_dcae_cs_schema():
+    res = requests.get(
+        "https://gerrit.onap.org/r/gitweb?p=dcaegen2/platform/cli.git;a=blob_plain;f=component-json-schemas/component-specification/dcae-cli-v2/component-spec-schema.json;hb=HEAD"
+    )
+    return res.json()
+
+
+def _get_format_version(target_name, data_formats):
+    """
+    search through the data formats for name, make sure we have it, and retrieve the version
+    """
+    # the df must exist, since the data formats were generated from the same metadata, or dataformats call blew up.
+    # So we don't do error checking here
+    for df in data_formats:
+        if df["self"]["name"] == target_name:
+            return df["self"]["version"]
+
+
+def _generate_spec(model_name, meta, dcae_cs_schema, data_formats, docker_uri):
+    """
+    Function that generates the component spec from the model metadata and docker info
+    Broken out to be unit-testable.
+    """
+
+    spec = {
+        "self": {
+            "version": "1.0.0",  # hopefully we get this from somewhere and not hardcode this
+            "name": model_name,
+            "description": "Automatically generated from Acumos model",
+            "component_type": "docker",
+        },
+        "services": {"calls": [], "provides": []},
+        "streams": {"subscribes": [], "publishes": []},
+        "parameters": [],
+        "auxilary": {"healthcheck": {"type": "http", "endpoint": "/healthcheck"}},
+        "artifacts": [{"type": "docker image", "uri": docker_uri}],
+    }
+
+    # from https://pypi.org/project/acumos-dcae-model-runner/
+    # each model method gets a subscruber and a publisher, using the methood name
+    pstype = "message_router"
+    for method in meta["methods"]:
+
+        df_in_name = meta["methods"][method]["input"]
+        subscriber = {
+            "config_key": "{0}_subscriber".format(method),
+            "format": df_in_name,
+            "version": _get_format_version(df_in_name, data_formats),
+            "type": pstype,
+        }
+
+        spec["streams"]["subscribes"].append(subscriber)
+
+        df_out_name = meta["methods"][method]["output"]
+
+        publisher = {
+            "config_key": "{0}_publisher".format(method),
+            "format": df_out_name,
+            "version": _get_format_version(df_out_name, data_formats),
+            "type": pstype,
+        }
+
+        spec["streams"]["publishes"].append(publisher)
+
+    # Validate that we have a valid spec
+    validate(instance=spec, schema=dcae_cs_schema)
+
+    return spec
+
+
+def generate_spec(model_repo_path, model_name, data_formats, docker_uri):
+    """
+    Generate and write the component spec to disk
+    Returns the spec
+    """
+    spec = _generate_spec(
+        model_name, utils.get_metadata(model_repo_path, model_name), _get_dcae_cs_schema(), data_formats, docker_uri
+    )
+    fname = "{0}_dcae_component_specification.json".format(model_name)
+    with open("{0}/{1}".format(model_repo_path, fname), "w") as f:
+        f.write(json.dumps(spec))
+
+    return spec
diff --git a/adapter/acumos/aoconversion/utils.py b/adapter/acumos/aoconversion/utils.py
new file mode 100644
index 0000000..a5aae75
--- /dev/null
+++ b/adapter/acumos/aoconversion/utils.py
@@ -0,0 +1,24 @@
+# ============LICENSE_START====================================================
+# org.onap.dcae
+# =============================================================================
+# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END======================================================
+
+import json
+
+
+def get_metadata(model_repo_path, model_name):
+    # for now, assume it's called "metadata.json"
+    return json.loads(open("{0}/{1}/metadata.json".format(model_repo_path, model_name), "r").read())