summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael F. Lamb <mike@datagrok.org>2017-09-06 10:28:14 -0700
committerMichael F. Lamb <mike@datagrok.org>2017-09-06 10:28:14 -0700
commit1cd1014b2919cd5b608e8d2d1bbb0c4011067b12 (patch)
tree8b57bf588ff2d7e0c77246b5b55ec51f7fc9fd6f
parent8b5e5b5a8e08eb58839972d7e3658bfbb78745a6 (diff)
Commit seed code for imagescanner
This imports the initial seed code for imagescanner. These files were imported from a tarball with the SHA1SUM f05839fedb9c78c1e37d5d45ee6c7b134049da63. From the contents of the tarball, some files which had executable permission set erroneously were adjusted and a .gitignore file was added. The result matches exactly the contents of the origin private repository at hash ddaaf67. Change-Id: I34463fa7b7a4a7f9b6804bef3a113340a90d6ecd Signed-off-by: Michael F. Lamb <mike@datagrok.org> Issue-Id: VVP-7
-rw-r--r--.gitignore89
-rw-r--r--Dockerfile65
-rw-r--r--LICENSE.TXT38
-rwxr-xr-xbin/imagescanner-frontend40
-rwxr-xr-xbin/imagescanner-image157
-rwxr-xr-xbin/imagescanner-worker55
-rwxr-xr-xbin/notifications-worker40
-rw-r--r--imagescanner/MANIFEST.in39
-rw-r--r--imagescanner/imagescanner/__init__.py42
-rw-r--r--imagescanner/imagescanner/frontend.py94
-rw-r--r--imagescanner/imagescanner/in_temp_dir.py59
-rw-r--r--imagescanner/imagescanner/tasks.py236
-rw-r--r--imagescanner/imagescanner/templates/form.html94
-rw-r--r--imagescanner/setup.py48
14 files changed, 1096 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..72364f9
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,89 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..24308f0
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,65 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+FROM alpine
+
+RUN apk add --no-cache \
+ clamav \
+ clamav-libunrar \
+ device-mapper \
+ file \
+ git \
+ multipath-tools \
+ openssh-client \
+ qemu \
+ rsyslog \
+ uwsgi-python3 \
+ wget \
+ ; :
+
+# Bootstrap the database since clamav is running for the first time
+RUN freshclam -v
+
+ENV IMAGESCANNER_LOGS_PATH=/var/log/imagescanner \
+ IMAGESCANNER_MOUNTPOINT=/mnt/imagescanner
+
+COPY imagescanner /opt/imagescanner
+COPY bin/* /usr/local/bin/
+RUN mkdir -p $IMAGESCANNER_MOUNTPOINT /run/clamav $IMAGESCANNER_LOGS_PATH; chown clamav /run/clamav
+RUN pip3 install flask requests celery[redis] /opt/imagescanner
+EXPOSE 80
diff --git a/LICENSE.TXT b/LICENSE.TXT
new file mode 100644
index 0000000..4b35a94
--- /dev/null
+++ b/LICENSE.TXT
@@ -0,0 +1,38 @@
+# -*- coding: utf8 -*-
+# ============LICENSE_START=======================================================
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+# \ No newline at end of file
diff --git a/bin/imagescanner-frontend b/bin/imagescanner-frontend
new file mode 100755
index 0000000..8718110
--- /dev/null
+++ b/bin/imagescanner-frontend
@@ -0,0 +1,40 @@
+#!/bin/sh
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+exec uwsgi --master --http-socket 0.0.0.0:80 --manage-script-name --mount /=imagescanner.frontend:app --need-plugin=python3
diff --git a/bin/imagescanner-image b/bin/imagescanner-image
new file mode 100755
index 0000000..966db52
--- /dev/null
+++ b/bin/imagescanner-image
@@ -0,0 +1,157 @@
+#!/bin/sh
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+set -e
+
+usage() {
+ cat <<-EOF
+ Usage: $0 DISK_IMAGE
+
+ Mount DISK_IMAGE and scan it, optionally decompressing first.
+
+ DISK_IMAGE must be a qcow or raw disk image.
+
+ GZip-compressed images will be decompressed in-place.
+
+ Environment variable IMAGESCANNER_MOUNTPOINT controls where the image
+ will be mounted while scan is in progress.
+ EOF
+}
+
+scan_image_dir() {
+ clamscan -r "$1"
+ return $?
+}
+
+image="$1"
+[ "$IMAGESCANNER_MOUNTPOINT" ] || export IMAGESCANNER_MOUNTPOINT="/mnt/imagescanner"
+[ -d "$IMAGESCANNER_MOUNTPOINT" ] || mkdir -p "$IMAGESCANNER_MOUNTPOINT"
+
+[ -e "$image" ] || {
+ echo "Error: image not found: $image"
+ exit 1
+}
+
+if [ "${image##*.}" = "gz" ]; then
+ echo "Decompressing image $image..."
+ gunzip "$image"
+ image="${image%.gz}"
+fi
+
+# Hueristic for determining image type:
+# 1. ask "file"
+# 2. failing that, check file extension
+echo "Detecting image type for $image..."
+detectedtype="$(file -b $image)"
+case "$detectedtype" in
+ "QEMU QCOW Image"*) imagetype=qcow ;;
+ "ISO 9660 CD-ROM"*) imagetype=iso ;;
+ "DOS/MBR boot sector"*) imagetype=img ;; # "isohybrid" bootable CD / disk image
+ *) echo "Could not detect image type by inspection; guessing by extension..."
+ # unknown string from 'file'; try by extension
+ case "${image##*.}" in
+ "qcow"|"qcow2") imagetype=qcow ;;
+ "iso") imagetype=iso ;;
+ "img") imagetype=img ;;
+ *)
+ echo "Error: $image has unknown image type: $detectedtype"
+ exit 3
+ ;;
+ esac
+ ;;
+esac
+echo "Detected $detectedtype: $imagetype"
+
+status=0
+case "$imagetype" in
+ qcow)
+ echo "Processing qcow image $image..."
+ qemu-nbd -rc /dev/nbd0 "$image"
+ partitions=$(kpartx -ravs /dev/nbd0 | cut -d' ' -f3)
+ for partition in $partitions
+ do
+ [ -e "/dev/mapper/$partition" ] || continue # nullglob
+ echo "Mounting qcow partition $image/$partition..."
+ mount -o ro "/dev/mapper/$partition" "$IMAGESCANNER_MOUNTPOINT"
+ echo "Scanning mounted image..."
+ scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$?
+ echo "Unmounting..."
+ umount "$IMAGESCANNER_MOUNTPOINT"
+ done
+ echo "Disconnecting NBD device..."
+ dmsetup remove $partitions
+ kpartx -vd /dev/nbd0
+ qemu-nbd -d /dev/nbd0
+ ;;
+
+ img)
+ echo "Processing raw image $image..."
+ partitions=$(kpartx -ravs $image | cut -d' ' -f3)
+ for partition in $partitions
+ do
+ [ -e "/dev/mapper/$partition" ] || continue # nullglob
+ echo "Mounting raw image partition $image/$partition..."
+ mount -o ro "/dev/mapper/$partition" "$IMAGESCANNER_MOUNTPOINT"
+ echo "Scanning mounted image..."
+ scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$?
+ echo "Unmounting..."
+ umount "$IMAGESCANNER_MOUNTPOINT"
+ done
+ echo "Disconnecting loopback device..."
+ # this is unnecessary on my host; why is it needed in a container?
+ dmsetup remove $partitions
+ kpartx -vd $image
+ ;;
+
+ iso)
+ echo "Processing iso image $image..."
+ mount -o loop,ro "$image" "$IMAGESCANNER_MOUNTPOINT"
+ echo "Scanning mounted image..."
+ scan_image_dir "$IMAGESCANNER_MOUNTPOINT" || status=$?
+ echo "Unmounting..."
+ umount "$IMAGESCANNER_MOUNTPOINT"
+ ;;
+
+esac
+echo "Done scanning $image."
+
+if [ "$status" != "0" ]; then
+ echo "WARNING: A scan reported a failure result."
+ exit $status
+fi
diff --git a/bin/imagescanner-worker b/bin/imagescanner-worker
new file mode 100755
index 0000000..7253738
--- /dev/null
+++ b/bin/imagescanner-worker
@@ -0,0 +1,55 @@
+#!/bin/sh
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+set -e
+
+# Verify host has loaded required modules
+for module in nbd isofs; do
+ lsmod | grep $module >/dev/null || echo "WARNING: $module kernel module not loaded."
+done
+
+# Run the update daemon in the background, if needed
+if ! [ -e "/run/clamav/freshclam.pid" ]; then
+ echo >&2 "Launching ClamAV virus database update daemon..."
+ freshclam -d -c 6
+fi
+
+# Run a celery worker for the scans queue. Limit concurrency to 1.
+echo >&2 "Launching imagescanner worker..."
+exec celery -A imagescanner.tasks.celery_app worker -c 1 -Q scans -n scanworker@%h
diff --git a/bin/notifications-worker b/bin/notifications-worker
new file mode 100755
index 0000000..b3c7b7c
--- /dev/null
+++ b/bin/notifications-worker
@@ -0,0 +1,40 @@
+#!/bin/sh
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+exec celery -A imagescanner.tasks.celery_app worker -n notifyworker@%h
diff --git a/imagescanner/MANIFEST.in b/imagescanner/MANIFEST.in
new file mode 100644
index 0000000..5af8897
--- /dev/null
+++ b/imagescanner/MANIFEST.in
@@ -0,0 +1,39 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+recursive-include imagescanner/templates *
diff --git a/imagescanner/imagescanner/__init__.py b/imagescanner/imagescanner/__init__.py
new file mode 100644
index 0000000..83fc05f
--- /dev/null
+++ b/imagescanner/imagescanner/__init__.py
@@ -0,0 +1,42 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+import os
+from pathlib import Path
+LOGS_PATH = Path(os.environ['IMAGESCANNER_LOGS_PATH'])
+STATUSFILE = LOGS_PATH/'status.txt'
diff --git a/imagescanner/imagescanner/frontend.py b/imagescanner/imagescanner/frontend.py
new file mode 100644
index 0000000..e27648a
--- /dev/null
+++ b/imagescanner/imagescanner/frontend.py
@@ -0,0 +1,94 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from flask import (
+ Flask, request, redirect, send_from_directory, url_for, render_template,
+ )
+import re
+from . import STATUSFILE, LOGS_PATH
+from .tasks import celery_app, request_scan
+
+app = Flask(__name__)
+# app.config['TRAP_HTTP_EXCEPTIONS'] = True
+# app.config['TRAP_BAD_REQUEST_ERRORS'] = True
+celery_inspect = celery_app.control.inspect()
+
+
+@app.route('/imagescanner')
+def show_form():
+ # TODO: consider storing worker status/state directly in redis
+ try:
+ with STATUSFILE.open() as fp:
+ status = fp.read()
+ except FileNotFoundError:
+ status = '(No status information available)'
+
+ return render_template(
+ 'form.html',
+ channel=os.getenv('DEFAULT_SLACK_CHANNEL', ''),
+ status=status,
+ active=(job
+ for worker, jobs in (celery_inspect.active() or {}).items()
+ for job in jobs),
+ reserved=(job
+ for worker, jobs in (celery_inspect.reserved() or {}).items()
+ for job in jobs),
+ )
+
+
+@app.route('/imagescanner', methods=['POST'])
+def process_form():
+ # TODO: better sanitize form input
+ request_scan.delay(
+ request.form['repo'],
+ request.form['path'],
+ re.split(r'[\s,]+', request.form['notify']),
+ )
+ return redirect(url_for('show_form'))
+
+
+@app.route('/imagescanner/result/<string(length=64):hashval>')
+def show_result_log(hashval):
+ if '/' in hashval:
+ raise ValueError("Invalid character in hashval")
+ return send_from_directory(
+ LOGS_PATH,
+ "SecurityValidation-%s.txt" % hashval,
+ )
diff --git a/imagescanner/imagescanner/in_temp_dir.py b/imagescanner/imagescanner/in_temp_dir.py
new file mode 100644
index 0000000..eaf57d5
--- /dev/null
+++ b/imagescanner/imagescanner/in_temp_dir.py
@@ -0,0 +1,59 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+
+
+@contextmanager
+def in_temp_dir(*args, **kwargs):
+ """A context manager that creates a temporary directory and changes the
+ current working directory to it, for the duration of the block.
+
+ """
+ with TemporaryDirectory(*args, **kwargs) as workspace:
+ try:
+ cwd = os.getcwd()
+ except FileNotFoundError:
+ cwd = None
+ os.chdir(workspace)
+ yield workspace
+ if cwd:
+ os.chdir(cwd)
diff --git a/imagescanner/imagescanner/tasks.py b/imagescanner/imagescanner/tasks.py
new file mode 100644
index 0000000..3610373
--- /dev/null
+++ b/imagescanner/imagescanner/tasks.py
@@ -0,0 +1,236 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+
+import os
+import re
+import hashlib
+import datetime
+from subprocess import run
+from celery import Celery
+import requests
+from . import STATUSFILE, LOGS_PATH
+from .in_temp_dir import in_temp_dir
+
+
+celery_app = Celery(
+ broker='redis://redis',
+ backend='redis://redis',
+ )
+repo_re = re.compile(r'.*\.git$')
+direct_re = re.compile(r'http.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+image_re = re.compile(r'.*\.(?:img|iso|qcow2)(?:\.gz)?$')
+SLACK_TOKEN = os.getenv('SLACK_TOKEN')
+DOMAIN = os.getenv('DOMAIN')
+
+
+def sha256(path):
+ """Return the SHA256 checksum of the file at path"""
+ h = hashlib.new('sha256')
+ with open(path, 'rb') as fd:
+ for chunk in iter((lambda: fd.read(4096)), b''):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+@celery_app.task(queue='scans', ignore_result=True)
+@in_temp_dir()
+def request_scan(source, path, recipients):
+ """Retrieve and scan all partitions of (an) image(s), and notify of the
+ results.
+
+ source:
+ A git URL referencing a repository containing one or more images, or an
+ HTTP(S) URL referencing a single image.
+
+ path:
+ If source is a git url, this specifies a path within that repo to an
+ image. If omitted, all images found in the repo will be scanned.
+ If source is an http url, this is ignored.
+
+ recipients:
+ A list of places to deliver a notification when the image scan is
+ complete. Currently, this may include Slack usernames and Slack
+ channels.
+
+ This function assumes the current working directory is a safe workarea for
+ retrieving and manipulating images, but is decorated with in_temp_dir which
+ changes to a new temporary directory upon invocation.
+
+ """
+
+ # TODO printing to a status file is archaic and messy; let's use the python
+ # logging framework or storing status in redis instead.
+ with STATUSFILE.open('w') as statusfile:
+
+ print(
+ "Processing request {source} {path} in {workspace}".format(
+ source=source, path=path, workspace=os.getcwd()),
+ file=statusfile,
+ flush=True)
+
+ for image in retrieve_images(source, path):
+ print("- Image file: {}...".format(image),
+ file=statusfile, flush=True)
+ if not os.path.exists(image):
+ raise ValueError("Path not found: {}".format(image))
+
+ print("-- Checksumming...", file=statusfile, flush=True)
+ checksum = sha256(image)
+
+ print("-- Scanning...",
+ file=statusfile, flush=True)
+ logfile = LOGS_PATH / 'SecurityValidation-{}.txt'.format(checksum)
+
+ #for partition in image_partitions():
+ # result = scan_partition(partition)
+ with open(logfile, 'w') as fd:
+ print(datetime.datetime.utcnow().ctime(), "UTC", file=fd)
+ print("Launching image scan for {} from {} {}".format(
+ image, source, path), file=fd)
+ print("SHA256 checksum:", checksum, file=fd, flush=True)
+ result = run(
+ ['/usr/local/bin/imagescanner-image', image],
+ stdout=fd,
+ stderr=fd,
+ )
+
+ print("-- Scheduling notification (exit code:{})...".format(result.returncode),
+ file=statusfile, flush=True)
+
+ slack_notify.delay(
+ status="Success" if result.returncode == 0 else "Failure",
+ source=source,
+ filename=image,
+ checksum=checksum,
+ recipients=recipients,
+ )
+
+ print("-- Done.", file=statusfile, flush=True)
+
+ print("- All images processed.", file=statusfile, flush=True)
+
+def retrieve_images(source, path):
+ """Generate the filenames of one or multiple disk images as they are
+ retrieved from _source_.
+
+ See the docstring for request_scan for documentation of the source and path
+ arguments.
+
+ This function assumes the current working directory is a safe workarea for
+ retrieving and manipulating images.
+
+ """
+ if repo_re.match(source):
+ return retrieve_images_git(source, path)
+ elif direct_re.match(source):
+ return retrieve_image_direct(source)
+ else:
+ raise ValueError("Unknown source format {}".format(source))
+
+
+def retrieve_images_git(source, path):
+ run(['/usr/bin/git', 'clone',
+ '--depth', '1',
+ '--single-branch',
+ '--recursive',
+ source,
+ 'repo/'],
+ env={"GIT_SSH_COMMAND": " ".join([
+ "ssh",
+ "-i /root/.ssh/id_ed25519",
+ "-o StrictHostKeyChecking=no"])},
+ check=True,
+ )
+
+ if path:
+ yield os.path.join("repo", path)
+ return
+
+ for root, dirs, files in os.walk('repo'):
+ for name in files:
+ if image_re.match(name):
+ yield os.path.join(root, name)
+
+
+def retrieve_image_direct(source):
+ filename = re.search(r'[^/]*$', source).group(0)
+ with open(filename, 'wb') as fd:
+ r = requests.get(source, stream=True)
+ for chunk in r.iter_content(chunk_size=4096):
+ fd.write(chunk)
+ yield filename
+
+
+# FIXME the slack notification should go into a different queue than the image
+# requests so they don't get blocked by the scans.
+@celery_app.task(ignore_result=True)
+def slack_notify(status, source, filename, checksum, recipients):
+ if not SLACK_TOKEN:
+ print("No Slack token defined; skipping notification.")
+ return
+
+ # TODO replace this handrolled code with a nice slack client library
+
+ link = "http://{}/imagescanner/result/{}".format(DOMAIN, checksum)
+
+ if filename.startswith('repo/'):
+ filename = filename[5:]
+
+ payload = {
+ "username": "Disk Image Scanning Robot",
+ "icon_emoji": ":robot_face:",
+ "attachments": [{
+ "fallback": "Image scan log: {}".format(link),
+ "pretext": "Disk image scan completed",
+ "color": "#00ff00" if status.lower() == 'success' else "#ff0000",
+ "title": "Scan {} for {}".format(status, filename),
+ "title_link": link,
+ "fields": [{"title": t, "value": v, "short": s} for t, v, s in [
+ ("Source", source, True),
+ ("Filename", filename, True),
+ ("Checksum", checksum, False),
+ ]]
+ }]
+ }
+
+ for recipient in recipients:
+ requests.post(
+ "https://hooks.slack.com/services/%s" % SLACK_TOKEN,
+ json=dict(payload, channel=recipient),
+ )
diff --git a/imagescanner/imagescanner/templates/form.html b/imagescanner/imagescanner/templates/form.html
new file mode 100644
index 0000000..7d037f3
--- /dev/null
+++ b/imagescanner/imagescanner/templates/form.html
@@ -0,0 +1,94 @@
+<!--
+/* ============LICENSE_START=======================================================
+ * org.onap.vvp/image-scanner
+ * ===================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * ===================================================================
+ *
+ * Unless otherwise specified, all software contained herein is licensed
+ * under the Apache License, Version 2.0 (the “License”);
+ * you may not use this software except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ *
+ * Unless otherwise specified, all documentation contained herein is licensed
+ * under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+ * you may not use this documentation except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://creativecommons.org/licenses/by/4.0/
+ *
+ * Unless required by applicable law or agreed to in writing, documentation
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * ============LICENSE_END============================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+-->
+<!doctype html>
+<html>
+ <head>
+ <style type="text/css">
+ input { width: 30em; }
+ </style>
+ </head>
+ <body>
+ <form method="POST">
+ <p>
+ <input name="repo"> <label for="repo">Git Repo URL</label><br/>
+ <input name="path"> <label for="path">Path to image</label><br/>
+ <input name="notify" value="{{channel}}"> <label for="path">Slack users/channels to notify</label><br/>
+ <input type="submit" value="Submit"></p>
+ </form>
+ <h3>Executing:</h3>
+ <pre>
+ {% for job in active -%}
+{{ job.args }}
+ {% else -%}
+(None)
+ {% endfor -%}
+ </pre>
+ <h3>Status:</h3>
+ <pre>{{status}}</pre>
+ <h3>Pending:</h3>
+ <pre>
+ {% for job in reserved -%}
+{{ job.args }}
+ {% else -%}
+(None)
+ {% endfor -%}
+ </pre>
+ <script language="javascript">
+ for (const k of document.getElementsByTagName("input")) {
+ if (k.name == "") { continue; }
+ r = new RegExp("(?:(?:^|.*;\\s*)"+k.name+"\\s*\\=\\s*([^;]*).*$)|^.*$");
+ let v = document.cookie.replace(r, "$1");
+ if (v == "") { continue; }
+ k.value = decodeURIComponent(v);
+ }
+ document.forms[0].onsubmit = function(){
+ for (const k of this.getElementsByTagName("input")) {
+ if (k.name == "") { continue; }
+ document.cookie = (encodeURIComponent(k.name)
+ + "=" + encodeURIComponent(k.value)
+ + ";path=/imagescanner"
+ + ";expires=Tue, 19 Jan 2038 03:14:07 GMT");
+ }
+ return true;
+ }
+ </script>
+ </body>
+</html>
diff --git a/imagescanner/setup.py b/imagescanner/setup.py
new file mode 100644
index 0000000..f387e7d
--- /dev/null
+++ b/imagescanner/setup.py
@@ -0,0 +1,48 @@
+# ============LICENSE_START=======================================================
+# org.onap.vvp/image-scanner
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the “License”);
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the “License”);
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+#
+# ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#
+from setuptools import setup
+
+setup(
+ name='imagescanner',
+ packages=['imagescanner'],
+ include_package_data=True,
+ package_data={
+ 'imagescanner': ['templates/*'],
+ }
+ )