diff options
Diffstat (limited to 'imagescanner')
| -rw-r--r-- | imagescanner/MANIFEST.in | 39 | ||||
| -rw-r--r-- | imagescanner/imagescanner/__init__.py | 42 | ||||
| -rw-r--r-- | imagescanner/imagescanner/frontend.py | 94 | ||||
| -rw-r--r-- | imagescanner/imagescanner/in_temp_dir.py | 59 | ||||
| -rw-r--r-- | imagescanner/imagescanner/tasks.py | 236 | ||||
| -rw-r--r-- | imagescanner/imagescanner/templates/form.html | 94 | ||||
| -rw-r--r-- | imagescanner/setup.py | 48 |
7 files changed, 612 insertions, 0 deletions
diff --git a/imagescanner/MANIFEST.in b/imagescanner/MANIFEST.in new file mode 100644 index 0000000..5af8897 --- /dev/null +++ b/imagescanner/MANIFEST.in @@ -0,0 +1,39 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +recursive-include imagescanner/templates * diff --git a/imagescanner/imagescanner/__init__.py b/imagescanner/imagescanner/__init__.py new file mode 100644 index 0000000..83fc05f --- /dev/null +++ b/imagescanner/imagescanner/__init__.py @@ -0,0 +1,42 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +import os +from pathlib import Path +LOGS_PATH = Path(os.environ['IMAGESCANNER_LOGS_PATH']) +STATUSFILE = LOGS_PATH/'status.txt' diff --git a/imagescanner/imagescanner/frontend.py b/imagescanner/imagescanner/frontend.py new file mode 100644 index 0000000..e27648a --- /dev/null +++ b/imagescanner/imagescanner/frontend.py @@ -0,0 +1,94 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +from flask import ( + Flask, request, redirect, send_from_directory, url_for, render_template, + ) +import re +from . import STATUSFILE, LOGS_PATH +from .tasks import celery_app, request_scan + +app = Flask(__name__) +# app.config['TRAP_HTTP_EXCEPTIONS'] = True +# app.config['TRAP_BAD_REQUEST_ERRORS'] = True +celery_inspect = celery_app.control.inspect() + + +@app.route('/imagescanner') +def show_form(): + # TODO: consider storing worker status/state directly in redis + try: + with STATUSFILE.open() as fp: + status = fp.read() + except FileNotFoundError: + status = '(No status information available)' + + return render_template( + 'form.html', + channel=os.getenv('DEFAULT_SLACK_CHANNEL', ''), + status=status, + active=(job + for worker, jobs in (celery_inspect.active() or {}).items() + for job in jobs), + reserved=(job + for worker, jobs in (celery_inspect.reserved() or {}).items() + for job in jobs), + ) + + +@app.route('/imagescanner', methods=['POST']) +def process_form(): + # TODO: better sanitize form input + request_scan.delay( + request.form['repo'], + request.form['path'], + re.split(r'[\s,]+', request.form['notify']), + ) + return redirect(url_for('show_form')) + + +@app.route('/imagescanner/result/<string(length=64):hashval>') +def show_result_log(hashval): + if '/' in hashval: + raise ValueError("Invalid character in hashval") + return send_from_directory( + LOGS_PATH, + "SecurityValidation-%s.txt" % hashval, + ) diff --git a/imagescanner/imagescanner/in_temp_dir.py b/imagescanner/imagescanner/in_temp_dir.py new file mode 100644 index 0000000..eaf57d5 --- /dev/null +++ b/imagescanner/imagescanner/in_temp_dir.py @@ -0,0 +1,59 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +from contextlib import contextmanager +from tempfile import TemporaryDirectory + + +@contextmanager +def in_temp_dir(*args, **kwargs): + """A context manager that creates a temporary directory and changes the + current working directory to it, for the duration of the block. + + """ + with TemporaryDirectory(*args, **kwargs) as workspace: + try: + cwd = os.getcwd() + except FileNotFoundError: + cwd = None + os.chdir(workspace) + yield workspace + if cwd: + os.chdir(cwd) diff --git a/imagescanner/imagescanner/tasks.py b/imagescanner/imagescanner/tasks.py new file mode 100644 index 0000000..3610373 --- /dev/null +++ b/imagescanner/imagescanner/tasks.py @@ -0,0 +1,236 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +import re +import hashlib +import datetime +from subprocess import run +from celery import Celery +import requests +from . import STATUSFILE, LOGS_PATH +from .in_temp_dir import in_temp_dir + + +celery_app = Celery( + broker='redis://redis', + backend='redis://redis', + ) +repo_re = re.compile(r'.*\.git$') +direct_re = re.compile(r'http.*\.(?:img|iso|qcow2)(?:\.gz)?$') +image_re = re.compile(r'.*\.(?:img|iso|qcow2)(?:\.gz)?$') +SLACK_TOKEN = os.getenv('SLACK_TOKEN') +DOMAIN = os.getenv('DOMAIN') + + +def sha256(path): + """Return the SHA256 checksum of the file at path""" + h = hashlib.new('sha256') + with open(path, 'rb') as fd: + for chunk in iter((lambda: fd.read(4096)), b''): + h.update(chunk) + return h.hexdigest() + + +@celery_app.task(queue='scans', ignore_result=True) +@in_temp_dir() +def request_scan(source, path, recipients): + """Retrieve and scan all partitions of (an) image(s), and notify of the + results. + + source: + A git URL referencing a repository containing one or more images, or an + HTTP(S) URL referencing a single image. + + path: + If source is a git url, this specifies a path within that repo to an + image. If omitted, all images found in the repo will be scanned. + If source is an http url, this is ignored. + + recipients: + A list of places to deliver a notification when the image scan is + complete. Currently, this may include Slack usernames and Slack + channels. + + This function assumes the current working directory is a safe workarea for + retrieving and manipulating images, but is decorated with in_temp_dir which + changes to a new temporary directory upon invocation. + + """ + + # TODO printing to a status file is archaic and messy; let's use the python + # logging framework or storing status in redis instead. + with STATUSFILE.open('w') as statusfile: + + print( + "Processing request {source} {path} in {workspace}".format( + source=source, path=path, workspace=os.getcwd()), + file=statusfile, + flush=True) + + for image in retrieve_images(source, path): + print("- Image file: {}...".format(image), + file=statusfile, flush=True) + if not os.path.exists(image): + raise ValueError("Path not found: {}".format(image)) + + print("-- Checksumming...", file=statusfile, flush=True) + checksum = sha256(image) + + print("-- Scanning...", + file=statusfile, flush=True) + logfile = LOGS_PATH / 'SecurityValidation-{}.txt'.format(checksum) + + #for partition in image_partitions(): + # result = scan_partition(partition) + with open(logfile, 'w') as fd: + print(datetime.datetime.utcnow().ctime(), "UTC", file=fd) + print("Launching image scan for {} from {} {}".format( + image, source, path), file=fd) + print("SHA256 checksum:", checksum, file=fd, flush=True) + result = run( + ['/usr/local/bin/imagescanner-image', image], + stdout=fd, + stderr=fd, + ) + + print("-- Scheduling notification (exit code:{})...".format(result.returncode), + file=statusfile, flush=True) + + slack_notify.delay( + status="Success" if result.returncode == 0 else "Failure", + source=source, + filename=image, + checksum=checksum, + recipients=recipients, + ) + + print("-- Done.", file=statusfile, flush=True) + + print("- All images processed.", file=statusfile, flush=True) + +def retrieve_images(source, path): + """Generate the filenames of one or multiple disk images as they are + retrieved from _source_. + + See the docstring for request_scan for documentation of the source and path + arguments. + + This function assumes the current working directory is a safe workarea for + retrieving and manipulating images. + + """ + if repo_re.match(source): + return retrieve_images_git(source, path) + elif direct_re.match(source): + return retrieve_image_direct(source) + else: + raise ValueError("Unknown source format {}".format(source)) + + +def retrieve_images_git(source, path): + run(['/usr/bin/git', 'clone', + '--depth', '1', + '--single-branch', + '--recursive', + source, + 'repo/'], + env={"GIT_SSH_COMMAND": " ".join([ + "ssh", + "-i /root/.ssh/id_ed25519", + "-o StrictHostKeyChecking=no"])}, + check=True, + ) + + if path: + yield os.path.join("repo", path) + return + + for root, dirs, files in os.walk('repo'): + for name in files: + if image_re.match(name): + yield os.path.join(root, name) + + +def retrieve_image_direct(source): + filename = re.search(r'[^/]*$', source).group(0) + with open(filename, 'wb') as fd: + r = requests.get(source, stream=True) + for chunk in r.iter_content(chunk_size=4096): + fd.write(chunk) + yield filename + + +# FIXME the slack notification should go into a different queue than the image +# requests so they don't get blocked by the scans. +@celery_app.task(ignore_result=True) +def slack_notify(status, source, filename, checksum, recipients): + if not SLACK_TOKEN: + print("No Slack token defined; skipping notification.") + return + + # TODO replace this handrolled code with a nice slack client library + + link = "http://{}/imagescanner/result/{}".format(DOMAIN, checksum) + + if filename.startswith('repo/'): + filename = filename[5:] + + payload = { + "username": "Disk Image Scanning Robot", + "icon_emoji": ":robot_face:", + "attachments": [{ + "fallback": "Image scan log: {}".format(link), + "pretext": "Disk image scan completed", + "color": "#00ff00" if status.lower() == 'success' else "#ff0000", + "title": "Scan {} for {}".format(status, filename), + "title_link": link, + "fields": [{"title": t, "value": v, "short": s} for t, v, s in [ + ("Source", source, True), + ("Filename", filename, True), + ("Checksum", checksum, False), + ]] + }] + } + + for recipient in recipients: + requests.post( + "https://hooks.slack.com/services/%s" % SLACK_TOKEN, + json=dict(payload, channel=recipient), + ) diff --git a/imagescanner/imagescanner/templates/form.html b/imagescanner/imagescanner/templates/form.html new file mode 100644 index 0000000..7d037f3 --- /dev/null +++ b/imagescanner/imagescanner/templates/form.html @@ -0,0 +1,94 @@ +<!-- +/* ============LICENSE_START======================================================= + * org.onap.vvp/image-scanner + * =================================================================== + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * =================================================================== + * + * Unless otherwise specified, all software contained herein is licensed + * under the Apache License, Version 2.0 (the “License”); + * you may not use this software except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + * + * Unless otherwise specified, all documentation contained herein is licensed + * under the Creative Commons License, Attribution 4.0 Intl. (the “License”); + * you may not use this documentation except in compliance with the License. + * You may obtain a copy of the License at + * + * https://creativecommons.org/licenses/by/4.0/ + * + * Unless required by applicable law or agreed to in writing, documentation + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * ============LICENSE_END============================================ + * + * ECOMP is a trademark and service mark of AT&T Intellectual Property. + */ +--> +<!doctype html> +<html> + <head> + <style type="text/css"> + input { width: 30em; } + </style> + </head> + <body> + <form method="POST"> + <p> + <input name="repo"> <label for="repo">Git Repo URL</label><br/> + <input name="path"> <label for="path">Path to image</label><br/> + <input name="notify" value="{{channel}}"> <label for="path">Slack users/channels to notify</label><br/> + <input type="submit" value="Submit"></p> + </form> + <h3>Executing:</h3> + <pre> + {% for job in active -%} +{{ job.args }} + {% else -%} +(None) + {% endfor -%} + </pre> + <h3>Status:</h3> + <pre>{{status}}</pre> + <h3>Pending:</h3> + <pre> + {% for job in reserved -%} +{{ job.args }} + {% else -%} +(None) + {% endfor -%} + </pre> + <script language="javascript"> + for (const k of document.getElementsByTagName("input")) { + if (k.name == "") { continue; } + r = new RegExp("(?:(?:^|.*;\\s*)"+k.name+"\\s*\\=\\s*([^;]*).*$)|^.*$"); + let v = document.cookie.replace(r, "$1"); + if (v == "") { continue; } + k.value = decodeURIComponent(v); + } + document.forms[0].onsubmit = function(){ + for (const k of this.getElementsByTagName("input")) { + if (k.name == "") { continue; } + document.cookie = (encodeURIComponent(k.name) + + "=" + encodeURIComponent(k.value) + + ";path=/imagescanner" + + ";expires=Tue, 19 Jan 2038 03:14:07 GMT"); + } + return true; + } + </script> + </body> +</html> diff --git a/imagescanner/setup.py b/imagescanner/setup.py new file mode 100644 index 0000000..f387e7d --- /dev/null +++ b/imagescanner/setup.py @@ -0,0 +1,48 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +from setuptools import setup + +setup( + name='imagescanner', + packages=['imagescanner'], + include_package_data=True, + package_data={ + 'imagescanner': ['templates/*'], + } + ) |