Refactor download.py and remove not used files

Final commit in download refactoring series.

Issue-ID: OOM-1803

Change-Id: I5b71908903ef131b8c16d08225c4d2e2de655130
Signed-off-by: Milan Verespej <m.verespej@partner.samsung.com>

10 files changed, 134 insertions, 986 deletions

diff --git a/build/download/base.py b/build/download/base.py
deleted file mode 100644
index d8b44839..00000000
--- a/build/download/base.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-import concurrent.futures
-import os
-import progressbar
-import prettytable
-import requests
-from distutils.spawn import find_executable
-
-progressbar.streams.wrap_stdout()
-progressbar.streams.wrap_stderr()
-
-
-def load_list(item_list):
-    """
-    Parse list with items to be downloaded.
-    :param item_list: File with list of items (1 line per item)
-    :return: set of items from file
-    """
-    with open(item_list, 'r') as f:
-        return {item for item in (line.strip() for line in f)
-                if item and not item.startswith('#')}
-
-
-def init_progress(items_name):
-    progress_widgets = ['Downloading {}: '.format(items_name),
-                        progressbar.Bar(), ' ',
-                        progressbar.Percentage(), ' ',
-                        '(', progressbar.SimpleProgress(), ')']
-
-    progress = progressbar.ProgressBar(widgets=progress_widgets,
-                                       poll_rate=1.0,
-                                       redirect_stdout=True)
-    return progress
-
-
-def start_progress(progress, target_count, skipping, log):
-    log_skipping(skipping, log)
-    log.info("Initializing download. Takes a while.")
-
-    progress.max_value = target_count
-    progress.start()
-    progress.update(len(skipping))
-
-
-def log_skipping(skipping_iterable, logger):
-    for skipped in skipping_iterable:
-        logger.info('Skipping: {}'.format(skipped))
-
-
-def run_concurrent(workers, progress, fn, iterable, *args):
-    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
-        futures = [executor.submit(fn, item, *args) for item in iterable]
-        error_count = 0
-        for future in concurrent.futures.as_completed(futures):
-            error = future.exception()
-            if error:
-                error_count += 1
-                progress.update()
-            else:
-                progress.update(progress.value +1)
-    return error_count
-
-
-def finish_progress(progress, error_count, log):
-    progress.finish(dirty=error_count > 0)
-    log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
-
-def check_tool(name):
-    return find_executable(name)
-
-def save_to_file(dst, content):
-    """
-    Save downloaded byte content to file
-    :param dst: path to file to save content to
-    :param content: byte content of file
-    """
-    dst_dir = os.path.dirname(dst)
-    if not os.path.exists(dst_dir):
-        os.makedirs(dst_dir)
-    with open(dst, 'wb') as dst_file:
-        dst_file.write(content)
-
-def make_get_request(url):
-    req = requests.get(url)
-    req.raise_for_status()
-    return req
-
-def simple_check_table(target, missing):
-    table = prettytable.PrettyTable(['Name', 'Downloaded'])
-    table.align['Name'] = 'l'
-    for item in sorted(target):
-        table.add_row([item, item not in missing])
-    return table
-
diff --git a/build/download/docker_images.py b/build/download/docker_images.py
deleted file mode 100755
index d8138dd6..00000000
--- a/build/download/docker_images.py
+++ /dev/null
@@ -1,268 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-import argparse
-import concurrent.futures
-import docker
-import itertools
-import json
-import logging
-import os
-import prettytable
-import sys
-import threading
-from retrying import retry
-
-import base
-
-log = logging.getLogger(__name__)
-
-
-def image_filename(image_name):
-    """
-    Get a name of a file where image will be saved.
-    :param image_name: Name of the image from list
-    :return: Filename of the image
-    """
-    return '{}.tar'.format(image_name.replace(':', '_').replace('/', '_'))
-
-
-def image_registry_name(image_name):
-    """
-    Get the name as shown in local registry. Since some strings are not part of name
-    when using default registry e.g. docker.io
-    :param image_name: name of the image from the list
-    :return: name of the image as it is shown by docker
-    """
-    name = image_name
-
-    if name.startswith('docker.io/'):
-        name = name.replace('docker.io/', '')
-
-    if name.startswith('library/'):
-        name = name.replace('library/', '')
-
-    if ':' not in name.rsplit('/')[-1]:
-        name = '{}:latest'.format(name)
-
-    return name
-
-
-def not_pulled_images(docker_client, target_list):
-    """
-    Get set of images that are not pulled on local system.
-    :param docker_client: docker.client.DockerClient
-    :param target_list: list of images to look for
-    :return: (set) images that are not present on local system
-    """
-    pulled = set(itertools.chain.from_iterable((image.tags for image
-                                                in docker_client.images.list())))
-    return {image for image in target_list if image_registry_name(image) not in pulled}
-
-
-def not_saved(target_images, target_dir):
-    """
-    Get set of images that are not saved in target directory
-    :param target_images: List of images to check for
-    :param target_dir: Directory where those images should be
-    :return: (set) Images that are missing from target directory
-    """
-    return set(image for image in target_images
-               if not os.path.isfile('/'.join((target_dir, image_filename(image)))))
-
-
-def missing(docker_client, target_list, save, target_dir):
-    """
-    Get dictionary of images not present locally.
-    :param docker_client: docker.client.DockerClient for communication with docker
-    :param target_list: list of desired images
-    :param save: (boolean) check for saved images
-    :param target_dir: target directory for saved images
-    :return: Dictionary of missing images ('not_pulled', 'not_saved')
-    """
-    return {'not_pulled': not_pulled_images(docker_client, target_list),
-            'not_saved': not_saved(target_list, target_dir) if save else set()}
-
-
-def merge_dict_sets(dictionary):
-    return set.union(*dictionary.values())
-
-
-def check_table(check_list, missing, save):
-    table = prettytable.PrettyTable(['Image', 'Pulled', 'Saved'])
-    table.align['Image'] = 'l'
-    for image in sorted(check_list):
-        pulled = not image in missing['not_pulled']
-        download_state = [pulled]
-        if save:
-            # if not pulled save anyway
-            download_state.append(pulled and not image in missing['not_saved'])
-        else:
-            download_state.append('Not checked')
-        table.add_row([image] + download_state)
-    return table
-
-
-@retry(stop_max_attempt_number=5, wait_fixed=5000)
-def pull_image(docker_client, image_name):
-    """
-    Pull docker image.
-    :param docker_client: docker.client.DockerClient for communication with docker
-    :param image_name: name of the image to be pulled
-    :return: pulled image (image object)
-    :raises docker.errors.APIError: after unsuccessful retries
-    """
-    if ':' not in image_name.rsplit('/')[-1]:
-        image_name = '{}:latest'.format(image_name)
-    try:
-        image = docker_client.images.pull(image_name)
-        log.info('Image {} pulled'.format(image_name))
-        return image
-    except docker.errors.APIError as err:
-        log.warning('Failed: {}: {}. Retrying...'.format(image_name, err))
-        raise err
-
-
-def save_image(image_name, image, output_dir, docker_client=None):
-    """
-    Save image to tar.
-    :param output_dir: path to destination directory
-    :param image: image object from pull_image function
-    :param image_name: name of the image from list
-    :param docker_client: docker.client.DockerClient for communication with docker
-    :return: None
-    """
-    dst = '{}/{}'.format(output_dir, image_filename(image_name))
-    if not os.path.exists(output_dir):
-        os.makedirs(output_dir)
-    if not isinstance(image, docker.models.images.Image):
-        image = docker_client.images.get(image_name)
-    try:
-        with open(dst, 'wb') as f:
-            for chunk in image.save(named=image_registry_name(image_name)):
-                f.write(chunk)
-        log.info('Image {} saved as {}'.format(image_name, dst))
-    except Exception as err:
-        os.remove(dst)
-        raise err
-
-
-def download_docker_image(image, save, output_dir, docker_client):
-    """ Pull and save docker image from specified docker registry
-    :param docker_client: docker.client.DockerClient for communication with docker
-    :param image: image to be downloaded
-    :param save: boolean - save image to disk or skip saving
-    :param output_dir: directory where image will be saved
-    :return: None
-    """
-    log.info('Downloading image: {}'.format(image))
-    try:
-        pulled_image = pull_image(docker_client, image)
-        if save:
-            save_image(image, pulled_image, output_dir)
-    except Exception as err:
-        log.exception('Error downloading {}: {}'.format(image, err))
-        raise err
-
-
-def download(image_list, save, output_dir, check_mode, progress, workers=3):
-    """
-    Download images from list
-    :param image_list: list of images to be downloaded
-    :param save: whether images should be saved to disk
-    :param output_dir: directory where images will be saved
-    :param check_mode: only check for missing images. No download
-    :param progress_bar: progressbar.ProgressBar to show how far download is
-    :return: None
-    """
-    try:
-        # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB)
-        docker_client = docker.client.DockerClient(version='auto', timeout=300)
-    except docker.errors.DockerException as err:
-        log.exception('Error creating docker client. Check if is docker installed and running'
-                  ' or if you have right permissions.')
-        raise err
-
-    target_images = base.load_list(image_list)
-    missing_images = missing(docker_client, target_images, save, output_dir)
-
-    if check_mode:
-        log.info(check_table(target_images, missing_images, save))
-        return
-
-    skipping = target_images - merge_dict_sets(missing_images)
-
-    base.start_progress(progress, len(target_images), skipping, log)
-
-    # if pulling and save is True. Save every pulled image to assure parity
-    error_count = base.run_concurrent(workers, progress, download_docker_image, missing_images['not_pulled'],
-                                      save, output_dir, docker_client)
-    # only save those that are pulled already but not saved
-    error_count += base.run_concurrent(workers, progress, save_image,
-                                       missing_images['not_saved'] - missing_images['not_pulled'],
-                                       None, output_dir, docker_client)
-
-    base.finish_progress(progress, error_count, log)
-    if error_count > 0:
-        log.error('{} images were not downloaded'.format(error_count))
-        missing_images = missing(docker_client, target_images, save, output_dir)
-        log.info(check_table(merge_dict_sets(missing_images), missing_images, save))
-        raise RuntimeError()
-
-
-def run_cli():
-    parser = argparse.ArgumentParser(description='Download docker images from list')
-    parser.add_argument('image_list', metavar='image-list',
-                        help='File with list of images to download.')
-    parser.add_argument('--save', '-s', action='store_true', default=False,
-                        help='Save images (without it only pull is executed)')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Download destination')
-    parser.add_argument('--check', '-c', action='store_true', default=False,
-                        help='Check what is missing. No download.'
-                             'Use with combination with -s to check saved images as well.')
-    parser.add_argument('--debug', action='store_true', default=False,
-                        help='Turn on debug output')
-    parser.add_argument('--workers', type=int, default=3,
-                        help='Set maximum workers for parallel download (default: 3)')
-
-    args = parser.parse_args()
-
-    if args.debug:
-        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-    else:
-        logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    progress = base.init_progress('Docker images') if not args.check else None
-    try:
-        download(args.image_list, args.save, args.output_dir, args.check,
-                 progress, args.workers)
-    except docker.errors.DockerException:
-        log.exception('Irrecoverable error detected.')
-        sys.exit(1)
-    except RuntimeError as err:
-        log.exception(err)
-
-
-if __name__ == '__main__':
-    run_cli()
-
diff --git a/build/download/download.py b/build/download/download.py
index 0d8912ee..0af12989 100755
--- a/build/download/download.py
+++ b/build/download/download.py
@@ -20,44 +20,51 @@
 #   COPYRIGHT NOTICE ENDS HERE
 
 import argparse
+import datetime
 import logging
 import sys
-import datetime
 import timeit
 
-import base
-import docker_images
-import git_repos
-import http_files
-import npm_packages
-import pypi_packages
-import rpm_packages
+import docker_downloader
+import git_downloader
+import http_downloader
+import npm_downloader
+import pypi_downloader
+import rpm_downloader
 
 log = logging.getLogger(name=__name__)
 
+
 def parse_args():
-    parser=argparse.ArgumentParser(description='Download data from lists')
+    """
+    Parse command line arguments
+    :return: arguments
+    """
+    parser = argparse.ArgumentParser(description='Download data from lists')
     list_group = parser.add_argument_group()
     list_group.add_argument('--docker', action='append', nargs='+', default=[],
-                        metavar=('list', 'dir-name'),
-                        help='Docker type list. If second argument is specified '
-                             'it is treated as directory where images will be saved '
-                             'otherwise only pull operation is executed')
+                            metavar=('list', 'dir-name'),
+                            help='Docker type list. If second argument is specified '
+                                 'it is treated as directory where images will be saved '
+                                 'otherwise only pull operation is executed this can\'t '
+                                 'be mixed between multiple docker list specifications. '
+                                 'if one of the list does not have directory specified '
+                                 'all lists are only pulled!!!')
     list_group.add_argument('--http', action='append', nargs=2, default=[],
-                        metavar=('list', 'dir-name'),
-                        help='Http type list and directory to save downloaded files')
+                            metavar=('list', 'dir-name'),
+                            help='Http type list and directory to save downloaded files')
     list_group.add_argument('--npm', action='append', nargs=2, default=[],
-                        metavar=('list', 'dir-name'),
-                        help='npm type list and directory to save downloaded files')
+                            metavar=('list', 'dir-name'),
+                            help='npm type list and directory to save downloaded files')
     list_group.add_argument('--rpm', action='append', nargs=2, default=[],
-                        metavar=('list', 'dir-name'),
-                        help='rpm type list and directory to save downloaded files')
+                            metavar=('list', 'dir-name'),
+                            help='rpm type list and directory to save downloaded files')
     list_group.add_argument('--git', action='append', nargs=2, default=[],
-                        metavar=('list', 'dir-name'),
-                        help='git repo type list and directory to save downloaded files')
+                            metavar=('list', 'dir-name'),
+                            help='git repo type list and directory to save downloaded files')
     list_group.add_argument('--pypi', action='append', nargs=2, default=[],
-                        metavar=('list', 'dir-name'),
-                        help='pypi packages type list and directory to save downloaded files')
+                            metavar=('list', 'dir-name'),
+                            help='pypi packages type list and directory to save downloaded files')
     parser.add_argument('--npm-registry', default='https://registry.npmjs.org',
                         help='npm registry to use (default: https://registry.npmjs.org)')
     parser.add_argument('--check', '-c', action='store_true', default=False,
@@ -71,10 +78,77 @@ def parse_args():
         if getattr(args, arg):
             return args
 
-    parser.error('One of --docker, --npm, --http, --rpm, --git must be specified')
+    parser.error('One of --docker, --npm, --http, --rpm, --git or --pypi must be specified')
+
+
+def log_start(item_type):
+    """
+    Log starting message
+    :param item_type: type of resources
+    :return:
+    """
+    log.info('Starting download of {}.'.format(item_type))
+
+
+def handle_download(downloader, check_mode, errorred_lists, start_time):
+    """
+    Handle download of resources
+    :param downloader: downloader to use
+    :param check_mode: run in check mode (boolean)
+    :param errorred_lists: list of data types of failed lists
+    :param start_time: timeit.default_timer() right before download
+    :return: timeit.default_timer() at the end of download
+    """
+    if check_mode:
+        print(downloader.check_table)
+    else:
+        log_start(downloader.list_type)
+        try:
+            downloader.download()
+        except RuntimeError:
+            errorred_lists.append(downloader.list_type)
+    return log_time_interval(start_time, downloader.list_type)
+
+
+def handle_command_download(downloader_class, check_mode, errorred_lists, start_time, *args):
+    """
+    Handle download of resources where shell command is used
+    :param downloader_class: Class of command_downloader.CommandDownloader to use
+    :param check_mode: run in check mode (boolean)
+    :param errorred_lists: list of data types of failed lists
+    :param start_time: timeit.default_timer() right before download
+    :param args: arguments for downloader class initialization
+    :return: timeit.default_timer() at the end of download
+    """
+    try:
+        downloader = downloader_class(*args)
+        return handle_download(downloader, check_mode, errorred_lists, start_time)
+    except FileNotFoundError as err:
+        classname = type(downloader_class).__name__
+        log.exception('Error initializing: {}: {}'.format(classname, err))
+    return timeit.default_timer()
+
+
+def log_time_interval(start, resource_type=''):
+    """
+    Log how long the download took
+    :param start: timeit.default_timer() when interval started
+    :param resource_type: type of data that was downloaded. (empty string for whole download)
+    :return: timeit.default_timer() after logging
+    """
+    e_time = datetime.timedelta(seconds=timeit.default_timer() - start)
+    if resource_type:
+        msg = 'Download of {} took {}\n'.format(resource_type, e_time)
+    else:
+        msg = 'Execution ended. Total elapsed time {}'.format(e_time)
+    log.info(msg)
+    return timeit.default_timer()
 
 
 def run_cli():
+    if sys.version_info.major < 3:
+        log.error('Unfortunately Python 2 is not supported for data download.')
+        sys.exit(1)
     args = parse_args()
 
     console_handler = logging.StreamHandler(sys.stdout)
@@ -91,82 +165,44 @@ def run_cli():
     root_logger = logging.getLogger()
     root_logger.addHandler(console_handler)
 
-    list_with_errors = []
-    timer_start = timeit.default_timer()
-
-    for docker_list in args.docker:
-        log.info('Processing {}.'.format(docker_list[0]))
-        progress = None if args.check else base.init_progress('docker images')
-        save = False
-        if len(docker_list) > 1:
-            save = True
-        else:
-            docker_list.append(None)
-        try:
-            docker_images.download(docker_list[0], save,
-                                   docker_list[1], args.check, progress)
-        except RuntimeError:
-            list_with_errors.append(docker_list[0])
+    errorred_lists = []
+    timer_start = interval_start = timeit.default_timer()
 
-    for http_list in args.http:
-        progress = None if args.check else base.init_progress('http files')
-        log.info('Processing {}.'.format(http_list[0]))
-        try:
-            http_files.download(http_list[0], http_list[1], args.check,
-                                progress)
-        except RuntimeError:
-            list_with_errors.append(http_list[0])
+    if args.check:
+        log.info('Check mode. No download will be executed.')
 
-    for npm_list in args.npm:
-        progress = None if args.check else base.init_progress('npm packages')
-        log.info('Processing {}.'.format(npm_list[0]))
-        try:
-            npm_packages.download(npm_list[0], args.npm_registry, npm_list[1],
-                                  args.check, progress)
-        except RuntimeError:
-            list_with_errors.append(npm_list[0])
+    if args.docker:
+        save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False
+        docker = docker_downloader.DockerDownloader(save, *args.docker, workers=3)
+        interval_start = handle_download(docker, args.check, errorred_lists, interval_start)
 
-    for rpm_list in args.rpm:
-        if args.check:
-            log.info('Check mode for rpm packages is not implemented')
-            break
-        log.info('Processing {}.'.format(rpm_list[0]))
-        try:
-            rpm_packages.download(rpm_list[0], rpm_list[1])
-        except RuntimeError:
-            list_with_errors.append(rpm_list[0])
-
-    for git_list in args.git:
-        if args.check:
-            log.info('Check mode for git repositories is not implemented')
-            break
-        progress = None if args.check else base.init_progress('git repositories')
-        log.info('Processing {}.'.format(git_list[0]))
-        try:
-            git_repos.download(git_list[0], git_list[1], progress)
-        except RuntimeError:
-            list_with_errors.append(git_list[0])
-
-    for pypi_list in args.pypi:
-        if args.check:
-            log.info('Check mode for pypi packages is not implemented')
-            break
-        progress = None if args.check else base.init_progress('pypi packages')
-        log.info('Processing {}.'.format(pypi_list[0]))
-        try:
-            pypi_packages.download(pypi_list[0], pypi_list[1], progress)
-        except RuntimeError:
-            list_with_errors.append(pypi_list[0])
+    if args.http:
+        http = http_downloader.HttpDownloader(*args.http)
+        interval_start = handle_download(http, args.check, errorred_lists, interval_start)
 
-    e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start)
-    log.info(timeit.default_timer() - timer_start)
-    log.info('Execution ended. Total elapsed time {}'.format(e_time))
+    if args.npm:
+        npm = npm_downloader.NpmDownloader(args.npm_registry, *args.npm)
+        interval_start = handle_download(npm, args.check, errorred_lists, interval_start)
 
-    if list_with_errors:
-        log.error('Errors encountered while processing these lists:'
-                  '\n{}'.format('\n'.join(list_with_errors)))
-        sys.exit(1)
+    if args.rpm:
+        interval_start = handle_command_download(rpm_downloader.RpmDownloader, args.check, errorred_lists,
+                                                 interval_start, *args.rpm)
+
+    if args.git:
+        interval_start = handle_command_download(git_downloader.GitDownloader, args.check, errorred_lists,
+                                                 interval_start, *args.git)
 
+    if args.pypi:
+        handle_command_download(pypi_downloader.PyPiDownloader, args.check, errorred_lists,
+                                interval_start, *args.pypi)
+
+    if not args.check:
+        log_time_interval(timer_start)
+
+    if errorred_lists:
+        log.error('Errors encountered while processing these types:'
+                  '\n{}'.format('\n'.join(errorred_lists)))
+        sys.exit(1)
 
 
 if __name__ == '__main__':
diff --git a/build/download/git_repos.py b/build/download/git_repos.py
deleted file mode 100755
index 1d8c2979..00000000
--- a/build/download/git_repos.py
+++ /dev/null
@@ -1,97 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HEREE
-
-import argparse
-import subprocess
-import shutil
-import logging
-import sys
-import os
-from retrying import retry
-
-import base
-
-log = logging.getLogger(name=__name__)
-
-@retry(stop_max_attempt_number=5, wait_fixed=5000)
-def clone_repo(dst, repo, branch=None):
-    if branch:
-        command = 'git clone -b {} --single-branch https://{} --bare {}'.format(branch, repo, dst)
-    else:
-        command = 'git clone https://{} --bare {}'.format(repo, dst)
-    log.info('Running: {}'.format(command))
-    log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode())
-    log.info('Downloaded: {}'.format(repo))
-
-
-def download(git_list, dst_dir, progress):
-    if not base.check_tool('git'):
-        log.error('ERROR: git is not installed')
-        progress.finish(dirty=True)
-        raise RuntimeError('git missing')
-
-    git_set = {tuple(item.split()) for item in base.load_list(git_list)}
-
-    error_count = 0
-
-    base.start_progress(progress, len(git_set), [], log)
-
-    for repo in git_set:
-        dst = '{}/{}'.format(dst_dir, repo[0])
-        if os.path.isdir(dst):
-            log.warning('Directory {} already exists. Repo probably present'.format(dst))
-            progress.update(progress.value + 1)
-            continue
-        try:
-            clone_repo(dst, *repo)
-            progress.update(progress.value + 1)
-        except subprocess.CalledProcessError as err:
-            if os.path.isdir(dst):
-                shutil.rmtree(dst)
-            log.exception(err.output.decode())
-            error_count += 1
-
-    base.finish_progress(progress, error_count, log)
-    if error_count > 0:
-        log.error('{} were not downloaded. Check logs for details'.format(error_count))
-        raise RuntimeError('Download unsuccessful')
-
-def run_cli():
-    parser = argparse.ArgumentParser(description='Download git repositories from list')
-    parser.add_argument('git_list', metavar='git-list',
-                        help='File with list of git repos to download.')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Download destination')
-
-    args = parser.parse_args()
-
-    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    progress = base.init_progress('git repositories')
-    try:
-        download(args.git_list, args.output_dir, progress)
-    except RuntimeError as err:
-        log.exception(err)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    run_cli()
diff --git a/build/download/http_files.py b/build/download/http_files.py
deleted file mode 100755
index c83158d6..00000000
--- a/build/download/http_files.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-import argparse
-import concurrent.futures
-import logging
-import os
-import sys
-from retrying import retry
-
-import base
-
-log = logging.getLogger(__name__)
-
-@retry(stop_max_attempt_number=5, wait_fixed=2000)
-def get_file(file_uri):
-    """
-    Get file from the Internet
-    :param file_uri: address of file
-    :return: byte content of file
-    """
-    if not file_uri.startswith('http'):
-        file_uri = 'http://' + file_uri
-    file_req = base.make_get_request(file_uri)
-    return file_req.content
-
-
-def download_file(file_uri, dst_dir):
-    """
-    Download http file and save it to file.
-    :param file_uri: http address of file
-    :param dst_dir: directory where file will be saved
-    """
-    log.info('Downloading: {}'.format(file_uri))
-    dst_path = '{}/{}'.format(dst_dir, file_uri.rsplit('//')[-1])
-    try:
-        file_content = get_file(file_uri)
-        base.save_to_file(dst_path, file_content)
-    except Exception as err:
-        if os.path.isfile(dst_path):
-            os.remove(dst_path)
-        log.error('Error downloading: {}: {}'.format(file_uri, err))
-        raise err
-    log.info('Downloaded: {}'.format(file_uri))
-
-
-def missing(file_set, dst_dir):
-    return {file for file in file_set if not os.path.isfile('{}/{}'.format(dst_dir, file))}
-
-
-def download(data_list, dst_dir, check, progress, workers=None):
-    """
-    Download files specified in data list
-    :param data_list: path to file with list
-    :param dst_dir: destination directory
-    :param check: boolean check mode
-    :param progress: progressbar.ProgressBar to monitor progress
-    :param workers: workers to use for parallel execution
-    :return: 0 if success else number of errors
-    """
-    file_set = base.load_list(data_list)
-    missing_files = missing(file_set, dst_dir)
-    target_count = len(file_set)
-
-    if check:
-        log.info(base.simple_check_table(file_set, missing_files))
-        return
-
-    skipping = file_set - missing_files
-
-    base.start_progress(progress, len(file_set), skipping, log)
-
-    error_count = base.run_concurrent(workers, progress, download_file, missing_files, dst_dir)
-
-    base.finish_progress(progress, error_count, log)
-    if error_count > 0:
-        log.error('{} files were not downloaded. Check log for specific failures.'.format(error_count))
-        raise RuntimeError()
-
-
-def run_cli():
-    """
-    Run as cli tool
-    """
-    parser = argparse.ArgumentParser(description='Download http files from list')
-    parser.add_argument('file_list', metavar='file-list',
-                        help='File with list of http files to download')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Destination directory for saving')
-    parser.add_argument('--check', '-c', action='store_true', default=False,
-                        help='Check mode')
-    parser.add_argument('--debug', action='store_true', default=False,
-                        help='Turn on debug output')
-    parser.add_argument('--workers', type=int, default=None,
-                        help='Set maximum workers for parallel download (default: cores * 5)')
-
-    args = parser.parse_args()
-
-    if args.debug:
-        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-    else:
-        logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    progress = base.init_progress('http files') if not args.check else None
-
-    try:
-        download(args.file_list, args.output_dir, args.check, progress, args.workers)
-    except RuntimeError:
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    run_cli()
-
diff --git a/build/download/npm_packages.py b/build/download/npm_packages.py
deleted file mode 100755
index 70c03ad8..00000000
--- a/build/download/npm_packages.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-import argparse
-import concurrent.futures
-import hashlib
-import logging
-import os
-import sys
-from retrying import retry
-
-import base
-
-log = logging.getLogger(name=__name__)
-
-
-@retry(stop_max_attempt_number=5, wait_fixed=5000)
-def get_npm(registry, npm_name, npm_version):
-    npm_url = '{}/{}/{}'.format(registry, npm_name, npm_version)
-    npm_req = base.make_get_request(npm_url)
-    npm_json = npm_req.json()
-    tarball_url = npm_json['dist']['tarball']
-    shasum = npm_json['dist']['shasum']
-    tarball_req = base.make_get_request(tarball_url)
-    tarball = tarball_req.content
-    if hashlib.sha1(tarball).hexdigest() == shasum:
-        return tarball
-    else:
-        raise Exception('{}@{}: Wrong checksum. Retrying...'.format(npm_name, npm_version))
-
-
-def download_npm(npm, registry, dst_dir):
-    log.info('Downloading: {}'.format(npm))
-    npm_name, npm_version = npm.split('@')
-    dst_path = '{}/{}-{}.tgz'.format(dst_dir, npm_name, npm_version)
-    try:
-        tarball = get_npm(registry, *npm.split('@'))
-        base.save_to_file(dst_path, tarball)
-    except Exception as err:
-        if os.path.isfile(dst_path):
-            os.remove(dst_path)
-        log.exception('Failed: {}'.format(npm))
-        raise err
-    log.info('Downloaded: {}'.format(npm))
-
-
-def missing(npm_set, dst_dir):
-    return {npm for npm in npm_set
-            if not os.path.isfile('{}/{}-{}.tgz'.format(dst_dir, *npm.split('@')))}
-
-
-def download(npm_list, registry, dst_dir, check_mode, progress=None, workers=None):
-    npm_set = base.load_list(npm_list)
-    target_count = len(npm_set)
-    missing_npms = missing(npm_set, dst_dir)
-
-    if check_mode:
-        log.info(base.simple_check_table(npm_set, missing_npms))
-        return 0
-
-    skipping = npm_set - missing_npms
-
-    base.start_progress(progress, len(npm_set), skipping, log)
-    error_count = base.run_concurrent(workers, progress, download_npm, missing_npms, registry, dst_dir)
-
-    base.finish_progress(progress, error_count, log)
-    if error_count > 0:
-        log.error('{} packages were not downloaded. Check log for specific failures.'.format(error_count))
-        raise RuntimeError()
-
-
-def run_cli():
-    parser = argparse.ArgumentParser(description='Download npm packages from list')
-    parser.add_argument('npm_list', metavar='npm-list',
-                        help='File with list of npm packages to download.')
-    parser.add_argument('--registry', '-r', default='https://registry.npmjs.org',
-                        help='Download destination')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Download destination')
-    parser.add_argument('--check', '-c', action='store_true', default=False,
-                        help='Check what is missing. No download.')
-    parser.add_argument('--debug', action='store_true', default=False,
-                        help='Turn on debug output')
-    parser.add_argument('--workers', type=int, default=None,
-                        help='Set maximum workers for parallel download (default: cores * 5)')
-
-    args = parser.parse_args()
-
-    if args.debug:
-        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-    else:
-        logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    progress = base.init_progress('npm packages') if not args.check else None
-    sys.exit(download(args.npm_list, args.registry, args.output_dir, args.check, progress,
-                      args.workers))
-
-
-if __name__ == '__main__':
-    run_cli()
-
diff --git a/build/download/pypi_packages.py b/build/download/pypi_packages.py
deleted file mode 100755
index 951003c5..00000000
--- a/build/download/pypi_packages.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-import argparse
-import logging
-import sys
-import subprocess
-import os
-from retrying import retry
-
-import base
-
-log = logging.getLogger(name=__name__)
-
-@retry(stop_max_attempt_number=5, wait_fixed=5000)
-def download_package(package_name, dst_dir):
-    command = 'pip download --dest {} {}'.format(dst_dir, package_name)
-    log.info('Running: {}'.format(command))
-    log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode())
-    log.info('Downloaded: {}'.format(package_name))
-
-
-def download(pypi_list, dst_dir, progress):
-    if not base.check_tool('pip'):
-        log.error('ERROR: pip is not installed')
-        progress.finish(dirty=True)
-        raise RuntimeError('pip missing')
-
-    pypi_set = base.load_list(pypi_list)
-
-    error_count = 0
-
-    base.start_progress(progress, len(pypi_set), [], log)
-
-    for package in pypi_set:
-        try:
-            download_package(package, dst_dir)
-        except subprocess.CalledProcessError as err:
-            log.exception(err.output.decode())
-            error_count += 1
-
-        progress.update(progress.value + 1)
-
-    base.finish_progress(progress, error_count, log)
-    if error_count > 0:
-        log.error('{} packages were not downloaded. Check logs for details'.format(error_count))
-        raise RuntimeError('Download unsuccesfull')
-
-
-def run_cli():
-    parser = argparse.ArgumentParser(description='Download git repositories from list')
-    parser.add_argument('pypi_list', metavar='pypi-list',
-                        help='File with list of pypi packages to download.')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Download destination')
-
-    args = parser.parse_args()
-
-    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    progress = base.init_progress('pypi packages')
-    try:
-        download(args.pypi_list, args.output_dir, progress)
-    except RuntimeError as err:
-        log.exception(err)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    run_cli()
diff --git a/build/download/requirements.txt b/build/download/requirements.txt
index 875f0c67..3eee2a2f 100644
--- a/build/download/requirements.txt
+++ b/build/download/requirements.txt
@@ -1,5 +1,3 @@
 docker==3.7.2
-futures==3.2.0; python_version == '2.7'
 prettytable==0.7.2
-progressbar2==3.39.3
 retrying==1.3.3
diff --git a/build/download/rpm_packages.py b/build/download/rpm_packages.py
deleted file mode 100755
index 732af0ea..00000000
--- a/build/download/rpm_packages.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#! /usr/bin/env python
-# -*- coding: utf-8 -*-
-
-#   COPYRIGHT NOTICE STARTS HERE
-
-#   Copyright 2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-#   COPYRIGHT NOTICE ENDS HERE
-
-import argparse
-import subprocess
-import logging
-import sys
-import os
-
-import base
-
-log = logging.getLogger(name=__name__)
-
-
-def download(rpm_list, dst_dir):
-    if not base.check_tool('yumdownloader'):
-        log.error('ERROR: yumdownloader is not installed')
-        raise RuntimeError('yumdownloader missing')
-
-    rpm_set = base.load_list(rpm_list)
-
-    command = 'yumdownloader --destdir={} {}'.format(dst_dir, ' '.join(rpm_set))
-    log.info('Running command: {}'.format(command))
-    try:
-        subprocess.check_call(command.split())
-    except subprocess.CalledProcessError as err:
-        log.exception(err.output)
-        raise err
-    log.info('Downloaded')
-
-
-def run_cli():
-    parser = argparse.ArgumentParser(description='Download rpm packages from list')
-    parser.add_argument('rpm_list', metavar='rpm-list',
-                        help='File with list of npm packages to download.')
-    parser.add_argument('--output-dir', '-o', default=os.getcwd(),
-                        help='Download destination')
-
-    args = parser.parse_args()
-
-    logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s')
-
-    try:
-        download(args.rpm_list, args.output_dir)
-    except (subprocess.CalledProcessError, RuntimeError):
-        sys.exit(1)
-
-
-
-if __name__ == '__main__':
-    run_cli()
diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst
index cfddcc99..8564a360 100644
--- a/docs/BuildGuide.rst
+++ b/docs/BuildGuide.rst
@@ -61,6 +61,9 @@ Subsequent steps are the same on both platforms:
     # install following packages
     yum install -y docker-ce-18.09.5 python-pip git createrepo expect nodejs npm jq
 
+    # install Python 3 (download scripts don't support Python 2 anymore)
+    yum install -y python36
+
     # twine package is needed by nexus blob build script
     pip install twine
 
@@ -77,7 +80,7 @@ Then it is necessary to clone all installer and build related repositories and p
     cd onap-offline
 
     # install required pip packages for download scripts
-    pip install -r ./build/download/requirements.txt
+    pip3 install -r ./build/download/requirements.txt
 
 Part 2. Download artifacts for offline installer
 ------------------------------------------------