diff options
Diffstat (limited to 'build/download/download.py')
| -rwxr-xr-x | build/download/download.py | 224 |
1 files changed, 130 insertions, 94 deletions
diff --git a/build/download/download.py b/build/download/download.py index 0d8912ee..0af12989 100755 --- a/build/download/download.py +++ b/build/download/download.py @@ -20,44 +20,51 @@ # COPYRIGHT NOTICE ENDS HERE import argparse +import datetime import logging import sys -import datetime import timeit -import base -import docker_images -import git_repos -import http_files -import npm_packages -import pypi_packages -import rpm_packages +import docker_downloader +import git_downloader +import http_downloader +import npm_downloader +import pypi_downloader +import rpm_downloader log = logging.getLogger(name=__name__) + def parse_args(): - parser=argparse.ArgumentParser(description='Download data from lists') + """ + Parse command line arguments + :return: arguments + """ + parser = argparse.ArgumentParser(description='Download data from lists') list_group = parser.add_argument_group() list_group.add_argument('--docker', action='append', nargs='+', default=[], - metavar=('list', 'dir-name'), - help='Docker type list. If second argument is specified ' - 'it is treated as directory where images will be saved ' - 'otherwise only pull operation is executed') + metavar=('list', 'dir-name'), + help='Docker type list. If second argument is specified ' + 'it is treated as directory where images will be saved ' + 'otherwise only pull operation is executed this can\'t ' + 'be mixed between multiple docker list specifications. ' + 'if one of the list does not have directory specified ' + 'all lists are only pulled!!!') list_group.add_argument('--http', action='append', nargs=2, default=[], - metavar=('list', 'dir-name'), - help='Http type list and directory to save downloaded files') + metavar=('list', 'dir-name'), + help='Http type list and directory to save downloaded files') list_group.add_argument('--npm', action='append', nargs=2, default=[], - metavar=('list', 'dir-name'), - help='npm type list and directory to save downloaded files') + metavar=('list', 'dir-name'), + help='npm type list and directory to save downloaded files') list_group.add_argument('--rpm', action='append', nargs=2, default=[], - metavar=('list', 'dir-name'), - help='rpm type list and directory to save downloaded files') + metavar=('list', 'dir-name'), + help='rpm type list and directory to save downloaded files') list_group.add_argument('--git', action='append', nargs=2, default=[], - metavar=('list', 'dir-name'), - help='git repo type list and directory to save downloaded files') + metavar=('list', 'dir-name'), + help='git repo type list and directory to save downloaded files') list_group.add_argument('--pypi', action='append', nargs=2, default=[], - metavar=('list', 'dir-name'), - help='pypi packages type list and directory to save downloaded files') + metavar=('list', 'dir-name'), + help='pypi packages type list and directory to save downloaded files') parser.add_argument('--npm-registry', default='https://registry.npmjs.org', help='npm registry to use (default: https://registry.npmjs.org)') parser.add_argument('--check', '-c', action='store_true', default=False, @@ -71,10 +78,77 @@ def parse_args(): if getattr(args, arg): return args - parser.error('One of --docker, --npm, --http, --rpm, --git must be specified') + parser.error('One of --docker, --npm, --http, --rpm, --git or --pypi must be specified') + + +def log_start(item_type): + """ + Log starting message + :param item_type: type of resources + :return: + """ + log.info('Starting download of {}.'.format(item_type)) + + +def handle_download(downloader, check_mode, errorred_lists, start_time): + """ + Handle download of resources + :param downloader: downloader to use + :param check_mode: run in check mode (boolean) + :param errorred_lists: list of data types of failed lists + :param start_time: timeit.default_timer() right before download + :return: timeit.default_timer() at the end of download + """ + if check_mode: + print(downloader.check_table) + else: + log_start(downloader.list_type) + try: + downloader.download() + except RuntimeError: + errorred_lists.append(downloader.list_type) + return log_time_interval(start_time, downloader.list_type) + + +def handle_command_download(downloader_class, check_mode, errorred_lists, start_time, *args): + """ + Handle download of resources where shell command is used + :param downloader_class: Class of command_downloader.CommandDownloader to use + :param check_mode: run in check mode (boolean) + :param errorred_lists: list of data types of failed lists + :param start_time: timeit.default_timer() right before download + :param args: arguments for downloader class initialization + :return: timeit.default_timer() at the end of download + """ + try: + downloader = downloader_class(*args) + return handle_download(downloader, check_mode, errorred_lists, start_time) + except FileNotFoundError as err: + classname = type(downloader_class).__name__ + log.exception('Error initializing: {}: {}'.format(classname, err)) + return timeit.default_timer() + + +def log_time_interval(start, resource_type=''): + """ + Log how long the download took + :param start: timeit.default_timer() when interval started + :param resource_type: type of data that was downloaded. (empty string for whole download) + :return: timeit.default_timer() after logging + """ + e_time = datetime.timedelta(seconds=timeit.default_timer() - start) + if resource_type: + msg = 'Download of {} took {}\n'.format(resource_type, e_time) + else: + msg = 'Execution ended. Total elapsed time {}'.format(e_time) + log.info(msg) + return timeit.default_timer() def run_cli(): + if sys.version_info.major < 3: + log.error('Unfortunately Python 2 is not supported for data download.') + sys.exit(1) args = parse_args() console_handler = logging.StreamHandler(sys.stdout) @@ -91,82 +165,44 @@ def run_cli(): root_logger = logging.getLogger() root_logger.addHandler(console_handler) - list_with_errors = [] - timer_start = timeit.default_timer() - - for docker_list in args.docker: - log.info('Processing {}.'.format(docker_list[0])) - progress = None if args.check else base.init_progress('docker images') - save = False - if len(docker_list) > 1: - save = True - else: - docker_list.append(None) - try: - docker_images.download(docker_list[0], save, - docker_list[1], args.check, progress) - except RuntimeError: - list_with_errors.append(docker_list[0]) + errorred_lists = [] + timer_start = interval_start = timeit.default_timer() - for http_list in args.http: - progress = None if args.check else base.init_progress('http files') - log.info('Processing {}.'.format(http_list[0])) - try: - http_files.download(http_list[0], http_list[1], args.check, - progress) - except RuntimeError: - list_with_errors.append(http_list[0]) + if args.check: + log.info('Check mode. No download will be executed.') - for npm_list in args.npm: - progress = None if args.check else base.init_progress('npm packages') - log.info('Processing {}.'.format(npm_list[0])) - try: - npm_packages.download(npm_list[0], args.npm_registry, npm_list[1], - args.check, progress) - except RuntimeError: - list_with_errors.append(npm_list[0]) + if args.docker: + save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False + docker = docker_downloader.DockerDownloader(save, *args.docker, workers=3) + interval_start = handle_download(docker, args.check, errorred_lists, interval_start) - for rpm_list in args.rpm: - if args.check: - log.info('Check mode for rpm packages is not implemented') - break - log.info('Processing {}.'.format(rpm_list[0])) - try: - rpm_packages.download(rpm_list[0], rpm_list[1]) - except RuntimeError: - list_with_errors.append(rpm_list[0]) - - for git_list in args.git: - if args.check: - log.info('Check mode for git repositories is not implemented') - break - progress = None if args.check else base.init_progress('git repositories') - log.info('Processing {}.'.format(git_list[0])) - try: - git_repos.download(git_list[0], git_list[1], progress) - except RuntimeError: - list_with_errors.append(git_list[0]) - - for pypi_list in args.pypi: - if args.check: - log.info('Check mode for pypi packages is not implemented') - break - progress = None if args.check else base.init_progress('pypi packages') - log.info('Processing {}.'.format(pypi_list[0])) - try: - pypi_packages.download(pypi_list[0], pypi_list[1], progress) - except RuntimeError: - list_with_errors.append(pypi_list[0]) + if args.http: + http = http_downloader.HttpDownloader(*args.http) + interval_start = handle_download(http, args.check, errorred_lists, interval_start) - e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) - log.info(timeit.default_timer() - timer_start) - log.info('Execution ended. Total elapsed time {}'.format(e_time)) + if args.npm: + npm = npm_downloader.NpmDownloader(args.npm_registry, *args.npm) + interval_start = handle_download(npm, args.check, errorred_lists, interval_start) - if list_with_errors: - log.error('Errors encountered while processing these lists:' - '\n{}'.format('\n'.join(list_with_errors))) - sys.exit(1) + if args.rpm: + interval_start = handle_command_download(rpm_downloader.RpmDownloader, args.check, errorred_lists, + interval_start, *args.rpm) + + if args.git: + interval_start = handle_command_download(git_downloader.GitDownloader, args.check, errorred_lists, + interval_start, *args.git) + if args.pypi: + handle_command_download(pypi_downloader.PyPiDownloader, args.check, errorred_lists, + interval_start, *args.pypi) + + if not args.check: + log_time_interval(timer_start) + + if errorred_lists: + log.error('Errors encountered while processing these types:' + '\n{}'.format('\n'.join(errorred_lists))) + sys.exit(1) if __name__ == '__main__': |