From 1cd1014b2919cd5b608e8d2d1bbb0c4011067b12 Mon Sep 17 00:00:00 2001 From: "Michael F. Lamb" Date: Wed, 6 Sep 2017 10:28:14 -0700 Subject: Commit seed code for imagescanner This imports the initial seed code for imagescanner. These files were imported from a tarball with the SHA1SUM f05839fedb9c78c1e37d5d45ee6c7b134049da63. From the contents of the tarball, some files which had executable permission set erroneously were adjusted and a .gitignore file was added. The result matches exactly the contents of the origin private repository at hash ddaaf67. Change-Id: I34463fa7b7a4a7f9b6804bef3a113340a90d6ecd Signed-off-by: Michael F. Lamb Issue-Id: VVP-7 --- .gitignore | 89 ++++++++++ Dockerfile | 65 +++++++ LICENSE.TXT | 38 +++++ bin/imagescanner-frontend | 40 +++++ bin/imagescanner-image | 157 +++++++++++++++++ bin/imagescanner-worker | 55 ++++++ bin/notifications-worker | 40 +++++ imagescanner/MANIFEST.in | 39 +++++ imagescanner/imagescanner/__init__.py | 42 +++++ imagescanner/imagescanner/frontend.py | 94 ++++++++++ imagescanner/imagescanner/in_temp_dir.py | 59 +++++++ imagescanner/imagescanner/tasks.py | 236 ++++++++++++++++++++++++++ imagescanner/imagescanner/templates/form.html | 94 ++++++++++ imagescanner/setup.py | 48 ++++++ 14 files changed, 1096 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE.TXT create mode 100755 bin/imagescanner-frontend create mode 100755 bin/imagescanner-image create mode 100755 bin/imagescanner-worker create mode 100755 bin/notifications-worker create mode 100644 imagescanner/MANIFEST.in create mode 100644 imagescanner/imagescanner/__init__.py create mode 100644 imagescanner/imagescanner/frontend.py create mode 100644 imagescanner/imagescanner/in_temp_dir.py create mode 100644 imagescanner/imagescanner/tasks.py create mode 100644 imagescanner/imagescanner/templates/form.html create mode 100644 imagescanner/setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..72364f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,89 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..24308f0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,65 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +FROM alpine + +RUN apk add --no-cache \ + clamav \ + clamav-libunrar \ + device-mapper \ + file \ + git \ + multipath-tools \ + openssh-client \ + qemu \ + rsyslog \ + uwsgi-python3 \ + wget \ + ; : + +# Bootstrap the database since clamav is running for the first time +RUN freshclam -v + +ENV IMAGESCANNER_LOGS_PATH=/var/log/imagescanner \ + IMAGESCANNER_MOUNTPOINT=/mnt/imagescanner + +COPY imagescanner /opt/imagescanner +COPY bin/* /usr/local/bin/ +RUN mkdir -p $IMAGESCANNER_MOUNTPOINT /run/clamav $IMAGESCANNER_LOGS_PATH; chown clamav /run/clamav +RUN pip3 install flask requests celery[redis] /opt/imagescanner +EXPOSE 80 diff --git a/LICENSE.TXT b/LICENSE.TXT new file mode 100644 index 0000000..4b35a94 --- /dev/null +++ b/LICENSE.TXT @@ -0,0 +1,38 @@ +# -*- coding: utf8 -*- +# ============LICENSE_START======================================================= +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# \ No newline at end of file diff --git a/bin/imagescanner-frontend b/bin/imagescanner-frontend new file mode 100755 index 0000000..8718110 --- /dev/null +++ b/bin/imagescanner-frontend @@ -0,0 +1,40 @@ +#!/bin/sh +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +exec uwsgi --master --http-socket 0.0.0.0:80 --manage-script-name --mount /=imagescanner.frontend:app --need-plugin=python3 diff --git a/bin/imagescanner-image b/bin/imagescanner-image new file mode 100755 index 0000000..966db52 --- /dev/null +++ b/bin/imagescanner-image @@ -0,0 +1,157 @@ +#!/bin/sh +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +set -e + +usage() { + cat <<-EOF + Usage: $0 DISK_IMAGE + + Mount DISK_IMAGE and scan it, optionally decompressing first. + + DISK_IMAGE must be a qcow or raw disk image. + + GZip-compressed images will be decompressed in-place. + + Environment variable IMAGESCANNER_MOUNTPOINT controls where the image + will be mounted while scan is in progress. + EOF +} + +scan_image_dir() { + clamscan -r "$1" + return $? +} + +image="$1" +[ "$IMAGESCANNER_MOUNTPOINT" ] || export IMAGESCANNER_MOUNTPOINT="/mnt/imagescanner" +[ -d "$IMAGESCANNER_MOUNTPOINT" ] || mkdir -p "$IMAGESCANNER_MOUNTPOINT" + +[ -e "$image" ] || { + echo "Error: image not found: $image" + exit 1 +} + +if [ "${image##*.}" = "gz" ]; then + echo "Decompressing image $image..." + gunzip "$image" + image="${image%.gz}" +fi + +# Hueristic for determining image type: +# 1. ask "file" +# 2. failing that, check file extension +echo "Detecting image type for $image..." +detectedtype="$(file -b $image)" +case "$detectedtype" in + "QEMU QCOW Image"*) imagetype=qcow ;; + "ISO 9660 CD-ROM"*) imagetype=iso ;; + "DOS/MBR boot sector"*) imagetype=img ;; # "isohybrid" bootable CD / disk image + *) echo "Could not detect image type by inspection; guessing by extension..." + # unknown string from 'file'; try by extension + case "${image##*.}" in + "qcow"|"qcow2") imagetype=qcow ;; + "iso") imagetype=iso ;; + "img") imagetype=img ;; + *) + echo "Error: $image has unknown image type: $detectedtype" + exit 3 + ;; + esac + ;; +esac +echo "Detected $detectedtype: $imagetype" + +status=0 +case "$imagetype" in + qcow) + echo "Processing qcow image $image..." + qemu-nbd -rc /dev/nbd0 "$image" + partitions=$(kpartx -ravs /dev/nbd0 | cut -d' ' -f3) + for partition in $partitions + do + [ -e "/dev/mapper/$partition" ] || continue # nullglob + echo "Mounting qcow partition $image/$partition..." + mount -o ro "/dev/mapper/$partition" "$IMAGESCANNER_MOUNTPOINT" + echo "Scanning mounted image..." + scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$? + echo "Unmounting..." + umount "$IMAGESCANNER_MOUNTPOINT" + done + echo "Disconnecting NBD device..." + dmsetup remove $partitions + kpartx -vd /dev/nbd0 + qemu-nbd -d /dev/nbd0 + ;; + + img) + echo "Processing raw image $image..." + partitions=$(kpartx -ravs $image | cut -d' ' -f3) + for partition in $partitions + do + [ -e "/dev/mapper/$partition" ] || continue # nullglob + echo "Mounting raw image partition $image/$partition..." + mount -o ro "/dev/mapper/$partition" "$IMAGESCANNER_MOUNTPOINT" + echo "Scanning mounted image..." + scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$? + echo "Unmounting..." + umount "$IMAGESCANNER_MOUNTPOINT" + done + echo "Disconnecting loopback device..." + # this is unnecessary on my host; why is it needed in a container? + dmsetup remove $partitions + kpartx -vd $image + ;; + + iso) + echo "Processing iso image $image..." + mount -o loop,ro "$image" "$IMAGESCANNER_MOUNTPOINT" + echo "Scanning mounted image..." + scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$? + echo "Unmounting..." + umount "$IMAGESCANNER_MOUNTPOINT" + ;; + +esac +echo "Done scanning $image." + +if [ "$status" != "0" ]; then + echo "WARNING: A scan reported a failure result." + exit $status +fi diff --git a/bin/imagescanner-worker b/bin/imagescanner-worker new file mode 100755 index 0000000..7253738 --- /dev/null +++ b/bin/imagescanner-worker @@ -0,0 +1,55 @@ +#!/bin/sh +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +set -e + +# Verify host has loaded required modules +for module in nbd isofs; do + lsmod | grep $module >/dev/null || echo "WARNING: $module kernel module not loaded." +done + +# Run the update daemon in the background, if needed +if ! [ -e "/run/clamav/freshclam.pid" ]; then + echo >&2 "Launching ClamAV virus database update daemon..." + freshclam -d -c 6 +fi + +# Run a celery worker for the scans queue. Limit concurrency to 1. +echo >&2 "Launching imagescanner worker..." +exec celery -A imagescanner.tasks.celery_app worker -c 1 -Q scans -n scanworker@%h diff --git a/bin/notifications-worker b/bin/notifications-worker new file mode 100755 index 0000000..b3c7b7c --- /dev/null +++ b/bin/notifications-worker @@ -0,0 +1,40 @@ +#!/bin/sh +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +exec celery -A imagescanner.tasks.celery_app worker -n notifyworker@%h diff --git a/imagescanner/MANIFEST.in b/imagescanner/MANIFEST.in new file mode 100644 index 0000000..5af8897 --- /dev/null +++ b/imagescanner/MANIFEST.in @@ -0,0 +1,39 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +recursive-include imagescanner/templates * diff --git a/imagescanner/imagescanner/__init__.py b/imagescanner/imagescanner/__init__.py new file mode 100644 index 0000000..83fc05f --- /dev/null +++ b/imagescanner/imagescanner/__init__.py @@ -0,0 +1,42 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +import os +from pathlib import Path +LOGS_PATH = Path(os.environ['IMAGESCANNER_LOGS_PATH']) +STATUSFILE = LOGS_PATH/'status.txt' diff --git a/imagescanner/imagescanner/frontend.py b/imagescanner/imagescanner/frontend.py new file mode 100644 index 0000000..e27648a --- /dev/null +++ b/imagescanner/imagescanner/frontend.py @@ -0,0 +1,94 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +from flask import ( + Flask, request, redirect, send_from_directory, url_for, render_template, + ) +import re +from . import STATUSFILE, LOGS_PATH +from .tasks import celery_app, request_scan + +app = Flask(__name__) +# app.config['TRAP_HTTP_EXCEPTIONS'] = True +# app.config['TRAP_BAD_REQUEST_ERRORS'] = True +celery_inspect = celery_app.control.inspect() + + +@app.route('/imagescanner') +def show_form(): + # TODO: consider storing worker status/state directly in redis + try: + with STATUSFILE.open() as fp: + status = fp.read() + except FileNotFoundError: + status = '(No status information available)' + + return render_template( + 'form.html', + channel=os.getenv('DEFAULT_SLACK_CHANNEL', ''), + status=status, + active=(job + for worker, jobs in (celery_inspect.active() or {}).items() + for job in jobs), + reserved=(job + for worker, jobs in (celery_inspect.reserved() or {}).items() + for job in jobs), + ) + + +@app.route('/imagescanner', methods=['POST']) +def process_form(): + # TODO: better sanitize form input + request_scan.delay( + request.form['repo'], + request.form['path'], + re.split(r'[\s,]+', request.form['notify']), + ) + return redirect(url_for('show_form')) + + +@app.route('/imagescanner/result/') +def show_result_log(hashval): + if '/' in hashval: + raise ValueError("Invalid character in hashval") + return send_from_directory( + LOGS_PATH, + "SecurityValidation-%s.txt" % hashval, + ) diff --git a/imagescanner/imagescanner/in_temp_dir.py b/imagescanner/imagescanner/in_temp_dir.py new file mode 100644 index 0000000..eaf57d5 --- /dev/null +++ b/imagescanner/imagescanner/in_temp_dir.py @@ -0,0 +1,59 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +from contextlib import contextmanager +from tempfile import TemporaryDirectory + + +@contextmanager +def in_temp_dir(*args, **kwargs): + """A context manager that creates a temporary directory and changes the + current working directory to it, for the duration of the block. + + """ + with TemporaryDirectory(*args, **kwargs) as workspace: + try: + cwd = os.getcwd() + except FileNotFoundError: + cwd = None + os.chdir(workspace) + yield workspace + if cwd: + os.chdir(cwd) diff --git a/imagescanner/imagescanner/tasks.py b/imagescanner/imagescanner/tasks.py new file mode 100644 index 0000000..3610373 --- /dev/null +++ b/imagescanner/imagescanner/tasks.py @@ -0,0 +1,236 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# + +import os +import re +import hashlib +import datetime +from subprocess import run +from celery import Celery +import requests +from . import STATUSFILE, LOGS_PATH +from .in_temp_dir import in_temp_dir + + +celery_app = Celery( + broker='redis://redis', + backend='redis://redis', + ) +repo_re = re.compile(r'.*\.git$') +direct_re = re.compile(r'http.*\.(?:img|iso|qcow2)(?:\.gz)?$') +image_re = re.compile(r'.*\.(?:img|iso|qcow2)(?:\.gz)?$') +SLACK_TOKEN = os.getenv('SLACK_TOKEN') +DOMAIN = os.getenv('DOMAIN') + + +def sha256(path): + """Return the SHA256 checksum of the file at path""" + h = hashlib.new('sha256') + with open(path, 'rb') as fd: + for chunk in iter((lambda: fd.read(4096)), b''): + h.update(chunk) + return h.hexdigest() + + +@celery_app.task(queue='scans', ignore_result=True) +@in_temp_dir() +def request_scan(source, path, recipients): + """Retrieve and scan all partitions of (an) image(s), and notify of the + results. + + source: + A git URL referencing a repository containing one or more images, or an + HTTP(S) URL referencing a single image. + + path: + If source is a git url, this specifies a path within that repo to an + image. If omitted, all images found in the repo will be scanned. + If source is an http url, this is ignored. + + recipients: + A list of places to deliver a notification when the image scan is + complete. Currently, this may include Slack usernames and Slack + channels. + + This function assumes the current working directory is a safe workarea for + retrieving and manipulating images, but is decorated with in_temp_dir which + changes to a new temporary directory upon invocation. + + """ + + # TODO printing to a status file is archaic and messy; let's use the python + # logging framework or storing status in redis instead. + with STATUSFILE.open('w') as statusfile: + + print( + "Processing request {source} {path} in {workspace}".format( + source=source, path=path, workspace=os.getcwd()), + file=statusfile, + flush=True) + + for image in retrieve_images(source, path): + print("- Image file: {}...".format(image), + file=statusfile, flush=True) + if not os.path.exists(image): + raise ValueError("Path not found: {}".format(image)) + + print("-- Checksumming...", file=statusfile, flush=True) + checksum = sha256(image) + + print("-- Scanning...", + file=statusfile, flush=True) + logfile = LOGS_PATH / 'SecurityValidation-{}.txt'.format(checksum) + + #for partition in image_partitions(): + # result = scan_partition(partition) + with open(logfile, 'w') as fd: + print(datetime.datetime.utcnow().ctime(), "UTC", file=fd) + print("Launching image scan for {} from {} {}".format( + image, source, path), file=fd) + print("SHA256 checksum:", checksum, file=fd, flush=True) + result = run( + ['/usr/local/bin/imagescanner-image', image], + stdout=fd, + stderr=fd, + ) + + print("-- Scheduling notification (exit code:{})...".format(result.returncode), + file=statusfile, flush=True) + + slack_notify.delay( + status="Success" if result.returncode == 0 else "Failure", + source=source, + filename=image, + checksum=checksum, + recipients=recipients, + ) + + print("-- Done.", file=statusfile, flush=True) + + print("- All images processed.", file=statusfile, flush=True) + +def retrieve_images(source, path): + """Generate the filenames of one or multiple disk images as they are + retrieved from _source_. + + See the docstring for request_scan for documentation of the source and path + arguments. + + This function assumes the current working directory is a safe workarea for + retrieving and manipulating images. + + """ + if repo_re.match(source): + return retrieve_images_git(source, path) + elif direct_re.match(source): + return retrieve_image_direct(source) + else: + raise ValueError("Unknown source format {}".format(source)) + + +def retrieve_images_git(source, path): + run(['/usr/bin/git', 'clone', + '--depth', '1', + '--single-branch', + '--recursive', + source, + 'repo/'], + env={"GIT_SSH_COMMAND": " ".join([ + "ssh", + "-i /root/.ssh/id_ed25519", + "-o StrictHostKeyChecking=no"])}, + check=True, + ) + + if path: + yield os.path.join("repo", path) + return + + for root, dirs, files in os.walk('repo'): + for name in files: + if image_re.match(name): + yield os.path.join(root, name) + + +def retrieve_image_direct(source): + filename = re.search(r'[^/]*$', source).group(0) + with open(filename, 'wb') as fd: + r = requests.get(source, stream=True) + for chunk in r.iter_content(chunk_size=4096): + fd.write(chunk) + yield filename + + +# FIXME the slack notification should go into a different queue than the image +# requests so they don't get blocked by the scans. +@celery_app.task(ignore_result=True) +def slack_notify(status, source, filename, checksum, recipients): + if not SLACK_TOKEN: + print("No Slack token defined; skipping notification.") + return + + # TODO replace this handrolled code with a nice slack client library + + link = "http://{}/imagescanner/result/{}".format(DOMAIN, checksum) + + if filename.startswith('repo/'): + filename = filename[5:] + + payload = { + "username": "Disk Image Scanning Robot", + "icon_emoji": ":robot_face:", + "attachments": [{ + "fallback": "Image scan log: {}".format(link), + "pretext": "Disk image scan completed", + "color": "#00ff00" if status.lower() == 'success' else "#ff0000", + "title": "Scan {} for {}".format(status, filename), + "title_link": link, + "fields": [{"title": t, "value": v, "short": s} for t, v, s in [ + ("Source", source, True), + ("Filename", filename, True), + ("Checksum", checksum, False), + ]] + }] + } + + for recipient in recipients: + requests.post( + "https://hooks.slack.com/services/%s" % SLACK_TOKEN, + json=dict(payload, channel=recipient), + ) diff --git a/imagescanner/imagescanner/templates/form.html b/imagescanner/imagescanner/templates/form.html new file mode 100644 index 0000000..7d037f3 --- /dev/null +++ b/imagescanner/imagescanner/templates/form.html @@ -0,0 +1,94 @@ + + + + + + + +
+

+
+
+
+

+
+

Executing:

+ 
+    {% for job in active -%}
+{{ job.args }}
+    {% else -%}
+(None)
+    {% endfor -%}
+
+

Status:

+ 
{{status}}
+

Pending:

+ 
+    {% for job in reserved -%}
+{{ job.args }}
+    {% else -%}
+(None)
+    {% endfor -%}
+
+ + + diff --git a/imagescanner/setup.py b/imagescanner/setup.py new file mode 100644 index 0000000..f387e7d --- /dev/null +++ b/imagescanner/setup.py @@ -0,0 +1,48 @@ +# ============LICENSE_START======================================================= +# org.onap.vvp/image-scanner +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the “License”); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the “License”); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +from setuptools import setup + +setup( + name='imagescanner', + packages=['imagescanner'], + include_package_data=True, + package_data={ + 'imagescanner': ['templates/*'], + } + ) -- cgit 1.2.3-korg