Commit seed code for imagescanner

This imports the initial seed code for imagescanner.

These files were imported from a tarball with the SHA1SUM
f05839fedb9c78c1e37d5d45ee6c7b134049da63.

From the contents of the tarball, some files which had executable
permission set erroneously were adjusted and a .gitignore file was
added. The result matches exactly the contents of the origin private
repository at hash ddaaf67.

Change-Id: I34463fa7b7a4a7f9b6804bef3a113340a90d6ecd
Signed-off-by: Michael F. Lamb <mike@datagrok.org>
Issue-Id: VVP-7

7 files changed, 612 insertions, 0 deletions

diff --git a/imagescanner/MANIFEST.in b/imagescanner/MANIFEST.in
new file mode 100644
index 0000000..5af8897
--- /dev/null
+++ b/imagescanner/MANIFEST.in
@@ -0,0 +1,39 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+recursive-include imagescanner/templates *
diff --git a/imagescanner/imagescanner/__init__.py b/imagescanner/imagescanner/__init__.py
new file mode 100644
index 0000000..83fc05f
--- /dev/null
+++ b/imagescanner/imagescanner/__init__.py
@@ -0,0 +1,42 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+import os
+from pathlib import Path
+LOGS_PATH = Path(os.environ['IMAGESCANNER_LOGS_PATH'])
+STATUSFILE = LOGS_PATH/'status.txt'
diff --git a/imagescanner/imagescanner/frontend.py b/imagescanner/imagescanner/frontend.py
new file mode 100644
index 0000000..e27648a
--- /dev/null
+++ b/imagescanner/imagescanner/frontend.py
@@ -0,0 +1,94 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from flask import (
+    Flask, request, redirect, send_from_directory, url_for, render_template,
+    )
+import re
+from . import STATUSFILE, LOGS_PATH
+from .tasks import celery_app, request_scan
+
+app = Flask(__name__)
+# app.config['TRAP_HTTP_EXCEPTIONS'] = True
+# app.config['TRAP_BAD_REQUEST_ERRORS'] = True
+celery_inspect = celery_app.control.inspect()
+
+
+@app.route('/imagescanner')
+def show_form():
+    # TODO: consider storing worker status/state directly in redis
+    try:
+        with STATUSFILE.open() as fp:
+            status = fp.read()
+    except FileNotFoundError:
+        status = '(No status information available)'
+
+    return render_template(
+        'form.html',
+        channel=os.getenv('DEFAULT_SLACK_CHANNEL', ''),
+        status=status,
+        active=(job
+            for worker, jobs in (celery_inspect.active() or {}).items()
+            for job in jobs),
+        reserved=(job
+            for worker, jobs in (celery_inspect.reserved() or {}).items()
+            for job in jobs),
+        )
+
+
+@app.route('/imagescanner', methods=['POST'])
+def process_form():
+    # TODO: better sanitize form input
+    request_scan.delay(
+        request.form['repo'],
+        request.form['path'],
+        re.split(r'[\s,]+', request.form['notify']),
+        )
+    return redirect(url_for('show_form'))
+
+
+@app.route('/imagescanner/result/<string(length=64):hashval>')
+def show_result_log(hashval):
+    if '/' in hashval:
+        raise ValueError("Invalid character in hashval")
+    return send_from_directory(
+        LOGS_PATH,
+        "SecurityValidation-%s.txt" % hashval,
+        )
diff --git a/imagescanner/imagescanner/in_temp_dir.py b/imagescanner/imagescanner/in_temp_dir.py
new file mode 100644
index 0000000..eaf57d5
--- /dev/null
+++ b/imagescanner/imagescanner/in_temp_dir.py
@@ -0,0 +1,59 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+
+
+@contextmanager
+def in_temp_dir(*args, **kwargs):
+    """A context manager that creates a temporary directory and changes the
+    current working directory to it, for the duration of the block.
+
+    """
+    with TemporaryDirectory(*args, **kwargs) as workspace:
+        try:
+            cwd = os.getcwd()
+        except FileNotFoundError:
+            cwd = None
+        os.chdir(workspace)
+        yield workspace
+        if cwd:
+            os.chdir(cwd)
diff --git a/imagescanner/imagescanner/tasks.py b/imagescanner/imagescanner/tasks.py
new file mode 100644
index 0000000..3610373
--- /dev/null
+++ b/imagescanner/imagescanner/tasks.py
@@ -0,0 +1,236 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+import re
+import hashlib
+import datetime
+from subprocess import run
+from celery import Celery
+import requests
+from . import STATUSFILE, LOGS_PATH
+from .in_temp_dir import in_temp_dir
+
+
+celery_app = Celery(
+    broker='redis://redis',
+    backend='redis://redis',
+    )
+repo_re = re.compile(r'.*\.git$')
+direct_re = re.compile(r'http.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+image_re = re.compile(r'.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+SLACK_TOKEN = os.getenv('SLACK_TOKEN')
+DOMAIN = os.getenv('DOMAIN')
+
+
+def sha256(path):
+    """Return the SHA256 checksum of the file at path"""
+    h = hashlib.new('sha256')
+    with open(path, 'rb') as fd:
+        for chunk in iter((lambda: fd.read(4096)), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+@celery_app.task(queue='scans', ignore_result=True)
+@in_temp_dir()
+def request_scan(source, path, recipients):
+    """Retrieve and scan all partitions of (an) image(s), and notify of the
+    results.
+
+    source:
+        A git URL referencing a repository containing one or more images, or an
+        HTTP(S) URL referencing a single image.
+
+    path:
+        If source is a git url, this specifies a path within that repo to an
+        image. If omitted, all images found in the repo will be scanned.
+        If source is an http url, this is ignored.
+
+    recipients:
+        A list of places to deliver a notification when the image scan is
+        complete. Currently, this may include Slack usernames and Slack
+        channels.
+
+    This function assumes the current working directory is a safe workarea for
+    retrieving and manipulating images, but is decorated with in_temp_dir which
+    changes to a new temporary directory upon invocation.
+
+    """
+
+    # TODO printing to a status file is archaic and messy; let's use the python
+    # logging framework or storing status in redis instead.
+    with STATUSFILE.open('w') as statusfile:
+
+        print(
+            "Processing request {source} {path} in {workspace}".format(
+                source=source, path=path, workspace=os.getcwd()),
+            file=statusfile,
+            flush=True)
+
+        for image in retrieve_images(source, path):
+            print("- Image file: {}...".format(image),
+                file=statusfile, flush=True)
+            if not os.path.exists(image):
+                raise ValueError("Path not found: {}".format(image))
+
+            print("-- Checksumming...", file=statusfile, flush=True)
+            checksum = sha256(image)
+
+            print("-- Scanning...",
+                file=statusfile, flush=True)
+            logfile = LOGS_PATH / 'SecurityValidation-{}.txt'.format(checksum)
+
+            #for partition in image_partitions():
+            #    result = scan_partition(partition)
+            with open(logfile, 'w') as fd:
+                print(datetime.datetime.utcnow().ctime(), "UTC", file=fd)
+                print("Launching image scan for {} from {} {}".format(
+                    image, source, path), file=fd)
+                print("SHA256 checksum:", checksum, file=fd, flush=True)
+                result = run(
+                    ['/usr/local/bin/imagescanner-image', image],
+                    stdout=fd,
+                    stderr=fd,
+                    )
+
+            print("-- Scheduling notification (exit code:{})...".format(result.returncode),
+                file=statusfile, flush=True)
+
+            slack_notify.delay(
+                status="Success" if result.returncode == 0 else "Failure",
+                source=source,
+                filename=image,
+                checksum=checksum,
+                recipients=recipients,
+                )
+
+            print("-- Done.", file=statusfile, flush=True)
+
+        print("- All images processed.", file=statusfile, flush=True)
+
+def retrieve_images(source, path):
+    """Generate the filenames of one or multiple disk images as they are
+    retrieved from _source_.
+
+    See the docstring for request_scan for documentation of the source and path
+    arguments.
+
+    This function assumes the current working directory is a safe workarea for
+    retrieving and manipulating images.
+
+    """
+    if repo_re.match(source):
+        return retrieve_images_git(source, path)
+    elif direct_re.match(source):
+        return retrieve_image_direct(source)
+    else:
+        raise ValueError("Unknown source format {}".format(source))
+
+
+def retrieve_images_git(source, path):
+    run(['/usr/bin/git', 'clone',
+         '--depth', '1',
+         '--single-branch',
+         '--recursive',
+         source,
+         'repo/'],
+        env={"GIT_SSH_COMMAND": " ".join([
+             "ssh",
+             "-i /root/.ssh/id_ed25519",
+             "-o StrictHostKeyChecking=no"])},
+        check=True,
+        )
+
+    if path:
+        yield os.path.join("repo", path)
+        return
+
+    for root, dirs, files in os.walk('repo'):
+        for name in files:
+            if image_re.match(name):
+                yield os.path.join(root, name)
+
+
+def retrieve_image_direct(source):
+    filename = re.search(r'[^/]*$', source).group(0)
+    with open(filename, 'wb') as fd:
+        r = requests.get(source, stream=True)
+        for chunk in r.iter_content(chunk_size=4096):
+            fd.write(chunk)
+    yield filename
+
+
+# FIXME the slack notification should go into a different queue than the image
+# requests so they don't get blocked by the scans.
+@celery_app.task(ignore_result=True)
+def slack_notify(status, source, filename, checksum, recipients):
+    if not SLACK_TOKEN:
+        print("No Slack token defined; skipping notification.")
+        return
+
+    # TODO replace this handrolled code with a nice slack client library
+
+    link = "http://{}/imagescanner/result/{}".format(DOMAIN, checksum)
+
+    if filename.startswith('repo/'):
+        filename = filename[5:]
+
+    payload = {
+        "username": "Disk Image Scanning Robot",
+        "icon_emoji": ":robot_face:",
+        "attachments": [{
+            "fallback": "Image scan log: {}".format(link),
+            "pretext": "Disk image scan completed",
+            "color": "#00ff00" if status.lower() == 'success' else "#ff0000",
+            "title": "Scan {} for {}".format(status, filename),
+            "title_link": link,
+            "fields": [{"title": t, "value": v, "short": s} for t, v, s in [
+                ("Source", source, True),
+                ("Filename", filename, True),
+                ("Checksum", checksum, False),
+                ]]
+            }]
+        }
+
+    for recipient in recipients:
+        requests.post(
+            "https://hooks.slack.com/services/%s" % SLACK_TOKEN,
+            json=dict(payload, channel=recipient),
+            )
diff --git a/imagescanner/imagescanner/templates/form.html b/imagescanner/imagescanner/templates/form.html
new file mode 100644
index 0000000..7d037f3
--- /dev/null
+++ b/imagescanner/imagescanner/templates/form.html
@@ -0,0 +1,94 @@
+<!--
+/* ============LICENSE_START=======================================================
+ * org.onap.vvp/image-scanner
+ * ===================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * ===================================================================
+ *
+ * Unless otherwise specified, all software contained herein is licensed
+ * under the Apache License, Version 2.0 (the “License”);
+ * you may not use this software except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *             http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ *
+ * Unless otherwise specified, all documentation contained herein is licensed
+ * under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+ * you may not use this documentation except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *             https://creativecommons.org/licenses/by/4.0/
+ *
+ * Unless required by applicable law or agreed to in writing, documentation
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * ============LICENSE_END============================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+-->
+<!doctype html>
+<html>
+  <head>
+    <style type="text/css">
+      input { width: 30em; }
+    </style>
+  </head>
+  <body>
+    <form method="POST">
+      <p>
+        <input name="repo"> <label for="repo">Git Repo URL</label><br/>
+        <input name="path"> <label for="path">Path to image</label><br/>
+        <input name="notify" value="{{channel}}"> <label for="path">Slack users/channels to notify</label><br/>
+        <input type="submit" value="Submit"></p>
+    </form>
+    <h3>Executing:</h3>
+    <pre>
+    {% for job in active -%}
+{{ job.args }}
+    {% else -%}
+(None)
+    {% endfor -%}
+    </pre>
+    <h3>Status:</h3>
+    <pre>{{status}}</pre>
+    <h3>Pending:</h3>
+    <pre>
+    {% for job in reserved -%}
+{{ job.args }}
+    {% else -%}
+(None)
+    {% endfor -%}
+    </pre>
+    <script language="javascript">
+      for (const k of document.getElementsByTagName("input")) {
+        if (k.name == "") { continue; }
+        r = new RegExp("(?:(?:^|.*;\\s*)"+k.name+"\\s*\\=\\s*([^;]*).*$)|^.*$");
+        let v = document.cookie.replace(r, "$1");
+        if (v == "") { continue; }
+        k.value = decodeURIComponent(v);
+      }
+      document.forms[0].onsubmit = function(){
+        for (const k of this.getElementsByTagName("input")) {
+          if (k.name == "") { continue; }
+          document.cookie = (encodeURIComponent(k.name)
+          + "=" + encodeURIComponent(k.value)
+          + ";path=/imagescanner"
+          + ";expires=Tue, 19 Jan 2038 03:14:07 GMT");
+        }
+        return true;
+      }
+    </script>
+  </body>
+</html>
diff --git a/imagescanner/setup.py b/imagescanner/setup.py
new file mode 100644
index 0000000..f387e7d
--- /dev/null
+++ b/imagescanner/setup.py
@@ -0,0 +1,48 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+from setuptools import setup
+
+setup(
+    name='imagescanner',
+    packages=['imagescanner'],
+    include_package_data=True,
+    package_data={
+        'imagescanner': ['templates/*'],
+        }
+    )