Add base download script

This script is supposed to be used for convenience when
downloading data from multiple lists at once.

Issue-ID: OOM-1803

Change-Id: I4031ed3650f7880883e299b43c79e6bfd08c886c
Signed-off-by: Milan Verespej <m.verespej@partner.samsung.com>

1 files changed, 158 insertions, 0 deletions

diff --git a/build/download/download.py b/build/download/download.py
new file mode 100755
index 00000000..ebce931a
--- /dev/null
+++ b/build/download/download.py
@@ -0,0 +1,158 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#   COPYRIGHT NOTICE STARTS HERE
+
+#   Copyright 2019 © Samsung Electronics Co., Ltd.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#   COPYRIGHT NOTICE ENDS HERE
+
+import argparse
+import logging
+import sys
+import datetime
+import timeit
+
+import base
+import docker_images
+import git_repos
+import http_files
+import npm_packages
+import rpm_packages
+
+log = logging.getLogger(name=__name__)
+
+def parse_args():
+    parser=argparse.ArgumentParser(description='Download data from lists')
+    list_group = parser.add_argument_group()
+    list_group.add_argument('--docker', action='append', nargs='+', default=[],
+                        metavar=('list', 'dir-name'),
+                        help='Docker type list. If second argument is specified '
+                             'it is treated as directory where images will be saved '
+                             'otherwise only pull operation is executed')
+    list_group.add_argument('--http', action='append', nargs=2, default=[],
+                        metavar=('list', 'dir-name'),
+                        help='Http type list and directory to save downloaded files')
+    list_group.add_argument('--npm', action='append', nargs=2, default=[],
+                        metavar=('list', 'dir-name'),
+                        help='npm type list and directory to save downloaded files')
+    list_group.add_argument('--rpm', action='append', nargs=2, default=[],
+                        metavar=('list', 'dir-name'),
+                        help='rpm type list and directory to save downloaded files')
+    list_group.add_argument('--git', action='append', nargs=2, default=[],
+                        metavar=('list', 'dir-name'),
+                        help='git repo type list and directory to save downloaded files')
+    parser.add_argument('--npm-registry', default='https://registry.npmjs.org',
+                        help='npm registry to use (default: https://registry.npmjs.org)')
+    parser.add_argument('--check', '-c', action='store_true', default=False,
+                        help='Check what is missing. No download.')
+    parser.add_argument('--debug', action='store_true', default=False,
+                        help='Turn on debug output')
+
+    args = parser.parse_args()
+
+    for arg in ('docker', 'npm', 'http', 'rpm', 'git'):
+        if getattr(args, arg):
+            return args
+
+    parser.error('One of --docker, --npm, --http, --rpm, --git must be specified')
+
+
+def run_cli():
+    args = parse_args()
+
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_formatter = logging.Formatter('%(message)s')
+    console_handler.setFormatter(console_formatter)
+    now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+    log_file = 'download_data-{}.log'.format(now)
+    file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s"
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format)
+    else:
+        logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format)
+    root_logger = logging.getLogger()
+    root_logger.addHandler(console_handler)
+
+    list_with_errors = []
+    timer_start = timeit.default_timer()
+
+    for docker_list in args.docker:
+        log.info('Processing {}.'.format(docker_list[0]))
+        progress = None if args.check else base.init_progress('docker images')
+        save = False
+        if len(docker_list) > 1:
+            save = True
+        else:
+            docker_list.append(None)
+        try:
+            docker_images.download(docker_list[0], save,
+                                   docker_list[1], args.check, progress)
+        except RuntimeError:
+            list_with_errors.append(docker_list[0])
+
+    for http_list in args.http:
+        progress = None if args.check else base.init_progress('http files')
+        log.info('Processing {}.'.format(http_list[0]))
+        try:
+            http_files.download(http_list[0], http_list[1], args.check,
+                                progress)
+        except RuntimeError:
+            list_with_errors.append(http_list[0])
+
+    for npm_list in args.npm:
+        progress = None if args.check else base.init_progress('npm packages')
+        log.info('Processing {}.'.format(npm_list[0]))
+        try:
+            npm_packages.download(npm_list[0], args.npm_registry, npm_list[1],
+                                  args.check, progress)
+        except RuntimeError:
+            list_with_errors.append(npm_list[0])
+
+    for rpm_list in args.rpm:
+        if args.check:
+            log.info('Check mode for rpm packages is not implemented')
+            break
+        log.info('Processing {}.'.format(rpm_list[0]))
+        try:
+            rpm_packages.download(rpm_list[0], rpm_list[1])
+        except RuntimeError:
+            list_with_errors.append(rpm_list[0])
+
+    for git_list in args.git:
+        if args.check:
+            log.info('Check mode for git repositories is not implemented')
+            break
+        progress = None if args.check else base.init_progress('git repositories')
+        log.info('Processing {}.'.format(git_list[0]))
+        try:
+            git_repos.download(git_list[0], git_list[1], progress)
+        except RuntimeError:
+            list_with_errors.append(git_list[0])
+
+    e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start)
+    log.info(timeit.default_timer() - timer_start)
+    log.info('Execution ended. Total elapsed time {}'.format(e_time))
+
+    if list_with_errors:
+        log.error('Errors encountered while processing these lists:'
+                  '\n{}'.format('\n'.join(list_with_errors)))
+        sys.exit(1)
+
+
+
+if __name__ == '__main__':
+    run_cli()