summaryrefslogtreecommitdiffstats
path: root/imagescanner
diff options
context:
space:
mode:
Diffstat (limited to 'imagescanner')
-rw-r--r--imagescanner/MANIFEST.in39
-rw-r--r--imagescanner/imagescanner/__init__.py42
-rw-r--r--imagescanner/imagescanner/frontend.py94
-rw-r--r--imagescanner/imagescanner/in_temp_dir.py59
-rw-r--r--imagescanner/imagescanner/tasks.py236
-rw-r--r--imagescanner/imagescanner/templates/form.html94
-rw-r--r--imagescanner/setup.py48
7 files changed, 612 insertions, 0 deletions
diff --git a/imagescanner/MANIFEST.in b/imagescanner/MANIFEST.in
new file mode 100644
index 0000000..5af8897
--- /dev/null
+++ b/imagescanner/MANIFEST.in
@@ -0,0 +1,39 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+recursive-include imagescanner/templates *
diff --git a/imagescanner/imagescanner/__init__.py b/imagescanner/imagescanner/__init__.py
new file mode 100644
index 0000000..83fc05f
--- /dev/null
+++ b/imagescanner/imagescanner/__init__.py
@@ -0,0 +1,42 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+import os
+from pathlib import Path
+LOGS_PATH = Path(os.environ['IMAGESCANNER_LOGS_PATH'])
+STATUSFILE = LOGS_PATH/'status.txt'
diff --git a/imagescanner/imagescanner/frontend.py b/imagescanner/imagescanner/frontend.py
new file mode 100644
index 0000000..e27648a
--- /dev/null
+++ b/imagescanner/imagescanner/frontend.py
@@ -0,0 +1,94 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from flask import (
+ Flask, request, redirect, send_from_directory, url_for, render_template,
+ )
+import re
+from . import STATUSFILE, LOGS_PATH
+from .tasks import celery_app, request_scan
+
+app = Flask(__name__)
+# app.config['TRAP_HTTP_EXCEPTIONS'] = True
+# app.config['TRAP_BAD_REQUEST_ERRORS'] = True
+celery_inspect = celery_app.control.inspect()
+
+
+@app.route('/imagescanner')
+def show_form():
+ # TODO: consider storing worker status/state directly in redis
+ try:
+ with STATUSFILE.open() as fp:
+ status = fp.read()
+ except FileNotFoundError:
+ status = '(No status information available)'
+
+ return render_template(
+ 'form.html',
+ channel=os.getenv('DEFAULT_SLACK_CHANNEL', ''),
+ status=status,
+ active=(job
+ for worker, jobs in (celery_inspect.active() or {}).items()
+ for job in jobs),
+ reserved=(job
+ for worker, jobs in (celery_inspect.reserved() or {}).items()
+ for job in jobs),
+ )
+
+
+@app.route('/imagescanner', methods=['POST'])
+def process_form():
+ # TODO: better sanitize form input
+ request_scan.delay(
+ request.form['repo'],
+ request.form['path'],
+ re.split(r'[\s,]+', request.form['notify']),
+ )
+ return redirect(url_for('show_form'))
+
+
+@app.route('/imagescanner/result/<string(length=64):hashval>')
+def show_result_log(hashval):
+ if '/' in hashval:
+ raise ValueError("Invalid character in hashval")
+ return send_from_directory(
+ LOGS_PATH,
+ "SecurityValidation-%s.txt" % hashval,
+ )
diff --git a/imagescanner/imagescanner/in_temp_dir.py b/imagescanner/imagescanner/in_temp_dir.py
new file mode 100644
index 0000000..eaf57d5
--- /dev/null
+++ b/imagescanner/imagescanner/in_temp_dir.py
@@ -0,0 +1,59 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+
+
+@contextmanager
+def in_temp_dir(*args, **kwargs):
+ """A context manager that creates a temporary directory and changes the
+ current working directory to it, for the duration of the block.
+
+ """
+ with TemporaryDirectory(*args, **kwargs) as workspace:
+ try:
+ cwd = os.getcwd()
+ except FileNotFoundError:
+ cwd = None
+ os.chdir(workspace)
+ yield workspace
+ if cwd:
+ os.chdir(cwd)
diff --git a/imagescanner/imagescanner/tasks.py b/imagescanner/imagescanner/tasks.py
new file mode 100644
index 0000000..3610373
--- /dev/null
+++ b/imagescanner/imagescanner/tasks.py
@@ -0,0 +1,236 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+import re
+import hashlib
+import datetime
+from subprocess import run
+from celery import Celery
+import requests
+from . import STATUSFILE, LOGS_PATH
+from .in_temp_dir import in_temp_dir
+
+
+celery_app = Celery(
+ broker='redis://redis',
+ backend='redis://redis',
+ )
+repo_re = re.compile(r'.*\.git$')
+direct_re = re.compile(r'http.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+image_re = re.compile(r'.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+SLACK_TOKEN = os.getenv('SLACK_TOKEN')
+DOMAIN = os.getenv('DOMAIN')
+
+
+def sha256(path):
+ """Return the SHA256 checksum of the file at path"""
+ h = hashlib.new('sha256')
+ with open(path, 'rb') as fd:
+ for chunk in iter((lambda: fd.read(4096)), b''):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+@celery_app.task(queue='scans', ignore_result=True)
+@in_temp_dir()
+def request_scan(source, path, recipients):
+ """Retrieve and scan all partitions of (an) image(s), and notify of the
+ results.
+
+ source:
+ A git URL referencing a repository containing one or more images, or an
+ HTTP(S) URL referencing a single image.
+
+ path:
+ If source is a git url, this specifies a path within that repo to an
+ image. If omitted, all images found in the repo will be scanned.
+ If source is an http url, this is ignored.
+
+ recipients:
+ A list of places to deliver a notification when the image scan is
+ complete. Currently, this may include Slack usernames and Slack
+ channels.
+
+ This function assumes the current working directory is a safe workarea for
+ retrieving and manipulating images, but is decorated with in_temp_dir which
+ changes to a new temporary directory upon invocation.
+
+ """
+
+ # TODO printing to a status file is archaic and messy; let's use the python
+ # logging framework or storing status in redis instead.
+ with STATUSFILE.open('w') as statusfile:
+
+ print(
+ "Processing request {source} {path} in {workspace}".format(
+ source=source, path=path, workspace=os.getcwd()),
+ file=statusfile,
+ flush=True)
+
+ for image in retrieve_images(source, path):
+ print("- Image file: {}...".format(image),
+ file=statusfile, flush=True)
+ if not os.path.exists(image):
+ raise ValueError("Path not found: {}".format(image))
+
+ print("-- Checksumming...", file=statusfile, flush=True)
+ checksum = sha256(image)
+
+ print("-- Scanning...",
+ file=statusfile, flush=True)
+ logfile = LOGS_PATH / 'SecurityValidation-{}.txt'.format(checksum)
+
+ #for partition in image_partitions():
+ # result = scan_partition(partition)
+ with open(logfile, 'w') as fd:
+ print(datetime.datetime.utcnow().ctime(), "UTC", file=fd)
+ print("Launching image scan for {} from {} {}".format(
+ image, source, path), file=fd)
+ print("SHA256 checksum:", checksum, file=fd, flush=True)
+ result = run(
+ ['/usr/local/bin/imagescanner-image', image],
+ stdout=fd,
+ stderr=fd,
+ )
+
+ print("-- Scheduling notification (exit code:{})...".format(result.returncode),
+ file=statusfile, flush=True)
+
+ slack_notify.delay(
+ status="Success" if result.returncode == 0 else "Failure",
+ source=source,
+ filename=image,
+ checksum=checksum,
+ recipients=recipients,
+ )
+
+ print("-- Done.", file=statusfile, flush=True)
+
+ print("- All images processed.", file=statusfile, flush=True)
+
+def retrieve_images(source, path):
+ """Generate the filenames of one or multiple disk images as they are
+ retrieved from _source_.
+
+ See the docstring for request_scan for documentation of the source and path
+ arguments.
+
+ This function assumes the current working directory is a safe workarea for
+ retrieving and manipulating images.
+
+ """
+ if repo_re.match(source):
+ return retrieve_images_git(source, path)
+ elif direct_re.match(source):
+ return retrieve_image_direct(source)
+ else:
+ raise ValueError("Unknown source format {}".format(source))
+
+
+def retrieve_images_git(source, path):
+ run(['/usr/bin/git', 'clone',
+ '--depth', '1',
+ '--single-branch',
+ '--recursive',
+ source,
+ 'repo/'],
+ env={"GIT_SSH_COMMAND": " ".join([
+ "ssh",
+ "-i /root/.ssh/id_ed25519",
+ "-o StrictHostKeyChecking=no"])},
+ check=True,
+ )
+
+ if path:
+ yield os.path.join("repo", path)
+ return
+
+ for root, dirs, files in os.walk('repo'):
+ for name in files:
+ if image_re.match(name):
+ yield os.path.join(root, name)
+
+
+def retrieve_image_direct(source):
+ filename = re.search(r'[^/]*$', source).group(0)
+ with open(filename, 'wb') as fd:
+ r = requests.get(source, stream=True)
+ for chunk in r.iter_content(chunk_size=4096):
+ fd.write(chunk)
+ yield filename
+
+
+# FIXME the slack notification should go into a different queue than the image
+# requests so they don't get blocked by the scans.
+@celery_app.task(ignore_result=True)
+def slack_notify(status, source, filename, checksum, recipients):
+ if not SLACK_TOKEN:
+ print("No Slack token defined; skipping notification.")
+ return
+
+ # TODO replace this handrolled code with a nice slack client library
+
+ link = "http://{}/imagescanner/result/{}".format(DOMAIN, checksum)
+
+ if filename.startswith('repo/'):
+ filename = filename[5:]
+
+ payload = {
+ "username": "Disk Image Scanning Robot",
+ "icon_emoji": ":robot_face:",
+ "attachments": [{
+ "fallback": "Image scan log: {}".format(link),
+ "pretext": "Disk image scan completed",
+ "color": "#00ff00" if status.lower() == 'success' else "#ff0000",
+ "title": "Scan {} for {}".format(status, filename),
+ "title_link": link,
+ "fields": [{"title": t, "value": v, "short": s} for t, v, s in [
+ ("Source", source, True),
+ ("Filename", filename, True),
+ ("Checksum", checksum, False),
+ ]]
+ }]
+ }
+
+ for recipient in recipients:
+ requests.post(
+ "https://hooks.slack.com/services/%s" % SLACK_TOKEN,
+ json=dict(payload, channel=recipient),
+ )
diff --git a/imagescanner/imagescanner/templates/form.html b/imagescanner/imagescanner/templates/form.html
new file mode 100644
index 0000000..7d037f3
--- /dev/null
+++ b/imagescanner/imagescanner/templates/form.html
@@ -0,0 +1,94 @@
+<!--
+/* ============LICENSE_START=======================================================
+ * org.onap.vvp/image-scanner
+ * ===================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * ===================================================================
+ *
+ * Unless otherwise specified, all software contained herein is licensed
+ * under the Apache License, Version 2.0 (the “License”);
+ * you may not use this software except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ *
+ * Unless otherwise specified, all documentation contained herein is licensed
+ * under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+ * you may not use this documentation except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://creativecommons.org/licenses/by/4.0/
+ *
+ * Unless required by applicable law or agreed to in writing, documentation
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * ============LICENSE_END============================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+-->
+<!doctype html>
+<html>
+ <head>
+ <style type="text/css">
+ input { width: 30em; }
+ </style>
+ </head>
+ <body>
+ <form method="POST">
+ <p>
+ <input name="repo"> <label for="repo">Git Repo URL</label><br/>
+ <input name="path"> <label for="path">Path to image</label><br/>
+ <input name="notify" value="{{channel}}"> <label for="path">Slack users/channels to notify</label><br/>
+ <input type="submit" value="Submit"></p>
+ </form>
+ <h3>Executing:</h3>
+ <pre>
+ {% for job in active -%}
+{{ job.args }}
+ {% else -%}
+(None)
+ {% endfor -%}
+ </pre>
+ <h3>Status:</h3>
+ <pre>{{status}}</pre>
+ <h3>Pending:</h3>
+ <pre>
+ {% for job in reserved -%}
+{{ job.args }}
+ {% else -%}
+(None)
+ {% endfor -%}
+ </pre>
+ <script language="javascript">
+ for (const k of document.getElementsByTagName("input")) {
+ if (k.name == "") { continue; }
+ r = new RegExp("(?:(?:^|.*;\\s*)"+k.name+"\\s*\\=\\s*([^;]*).*$)|^.*$");
+ let v = document.cookie.replace(r, "$1");
+ if (v == "") { continue; }
+ k.value = decodeURIComponent(v);
+ }
+ document.forms[0].onsubmit = function(){
+ for (const k of this.getElementsByTagName("input")) {
+ if (k.name == "") { continue; }
+ document.cookie = (encodeURIComponent(k.name)
+ + "=" + encodeURIComponent(k.value)
+ + ";path=/imagescanner"
+ + ";expires=Tue, 19 Jan 2038 03:14:07 GMT");
+ }
+ return true;
+ }
+ </script>
+ </body>
+</html>
diff --git a/imagescanner/setup.py b/imagescanner/setup.py
new file mode 100644
index 0000000..f387e7d
--- /dev/null
+++ b/imagescanner/setup.py
@@ -0,0 +1,48 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+from setuptools import setup
+
+setup(
+ name='imagescanner',
+ packages=['imagescanner'],
+ include_package_data=True,
+ package_data={
+ 'imagescanner': ['templates/*'],
+ }
+ )