diff options
Diffstat (limited to 'build/download/docker_images.py')
| -rwxr-xr-x | build/download/docker_images.py | 268 |
1 files changed, 0 insertions, 268 deletions
diff --git a/build/download/docker_images.py b/build/download/docker_images.py deleted file mode 100755 index d8138dd6..00000000 --- a/build/download/docker_images.py +++ /dev/null @@ -1,268 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -# COPYRIGHT NOTICE STARTS HERE - -# Copyright 2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# COPYRIGHT NOTICE ENDS HERE - - -import argparse -import concurrent.futures -import docker -import itertools -import json -import logging -import os -import prettytable -import sys -import threading -from retrying import retry - -import base - -log = logging.getLogger(__name__) - - -def image_filename(image_name): - """ - Get a name of a file where image will be saved. - :param image_name: Name of the image from list - :return: Filename of the image - """ - return '{}.tar'.format(image_name.replace(':', '_').replace('/', '_')) - - -def image_registry_name(image_name): - """ - Get the name as shown in local registry. Since some strings are not part of name - when using default registry e.g. docker.io - :param image_name: name of the image from the list - :return: name of the image as it is shown by docker - """ - name = image_name - - if name.startswith('docker.io/'): - name = name.replace('docker.io/', '') - - if name.startswith('library/'): - name = name.replace('library/', '') - - if ':' not in name.rsplit('/')[-1]: - name = '{}:latest'.format(name) - - return name - - -def not_pulled_images(docker_client, target_list): - """ - Get set of images that are not pulled on local system. - :param docker_client: docker.client.DockerClient - :param target_list: list of images to look for - :return: (set) images that are not present on local system - """ - pulled = set(itertools.chain.from_iterable((image.tags for image - in docker_client.images.list()))) - return {image for image in target_list if image_registry_name(image) not in pulled} - - -def not_saved(target_images, target_dir): - """ - Get set of images that are not saved in target directory - :param target_images: List of images to check for - :param target_dir: Directory where those images should be - :return: (set) Images that are missing from target directory - """ - return set(image for image in target_images - if not os.path.isfile('/'.join((target_dir, image_filename(image))))) - - -def missing(docker_client, target_list, save, target_dir): - """ - Get dictionary of images not present locally. - :param docker_client: docker.client.DockerClient for communication with docker - :param target_list: list of desired images - :param save: (boolean) check for saved images - :param target_dir: target directory for saved images - :return: Dictionary of missing images ('not_pulled', 'not_saved') - """ - return {'not_pulled': not_pulled_images(docker_client, target_list), - 'not_saved': not_saved(target_list, target_dir) if save else set()} - - -def merge_dict_sets(dictionary): - return set.union(*dictionary.values()) - - -def check_table(check_list, missing, save): - table = prettytable.PrettyTable(['Image', 'Pulled', 'Saved']) - table.align['Image'] = 'l' - for image in sorted(check_list): - pulled = not image in missing['not_pulled'] - download_state = [pulled] - if save: - # if not pulled save anyway - download_state.append(pulled and not image in missing['not_saved']) - else: - download_state.append('Not checked') - table.add_row([image] + download_state) - return table - - -@retry(stop_max_attempt_number=5, wait_fixed=5000) -def pull_image(docker_client, image_name): - """ - Pull docker image. - :param docker_client: docker.client.DockerClient for communication with docker - :param image_name: name of the image to be pulled - :return: pulled image (image object) - :raises docker.errors.APIError: after unsuccessful retries - """ - if ':' not in image_name.rsplit('/')[-1]: - image_name = '{}:latest'.format(image_name) - try: - image = docker_client.images.pull(image_name) - log.info('Image {} pulled'.format(image_name)) - return image - except docker.errors.APIError as err: - log.warning('Failed: {}: {}. Retrying...'.format(image_name, err)) - raise err - - -def save_image(image_name, image, output_dir, docker_client=None): - """ - Save image to tar. - :param output_dir: path to destination directory - :param image: image object from pull_image function - :param image_name: name of the image from list - :param docker_client: docker.client.DockerClient for communication with docker - :return: None - """ - dst = '{}/{}'.format(output_dir, image_filename(image_name)) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - if not isinstance(image, docker.models.images.Image): - image = docker_client.images.get(image_name) - try: - with open(dst, 'wb') as f: - for chunk in image.save(named=image_registry_name(image_name)): - f.write(chunk) - log.info('Image {} saved as {}'.format(image_name, dst)) - except Exception as err: - os.remove(dst) - raise err - - -def download_docker_image(image, save, output_dir, docker_client): - """ Pull and save docker image from specified docker registry - :param docker_client: docker.client.DockerClient for communication with docker - :param image: image to be downloaded - :param save: boolean - save image to disk or skip saving - :param output_dir: directory where image will be saved - :return: None - """ - log.info('Downloading image: {}'.format(image)) - try: - pulled_image = pull_image(docker_client, image) - if save: - save_image(image, pulled_image, output_dir) - except Exception as err: - log.exception('Error downloading {}: {}'.format(image, err)) - raise err - - -def download(image_list, save, output_dir, check_mode, progress, workers=3): - """ - Download images from list - :param image_list: list of images to be downloaded - :param save: whether images should be saved to disk - :param output_dir: directory where images will be saved - :param check_mode: only check for missing images. No download - :param progress_bar: progressbar.ProgressBar to show how far download is - :return: None - """ - try: - # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB) - docker_client = docker.client.DockerClient(version='auto', timeout=300) - except docker.errors.DockerException as err: - log.exception('Error creating docker client. Check if is docker installed and running' - ' or if you have right permissions.') - raise err - - target_images = base.load_list(image_list) - missing_images = missing(docker_client, target_images, save, output_dir) - - if check_mode: - log.info(check_table(target_images, missing_images, save)) - return - - skipping = target_images - merge_dict_sets(missing_images) - - base.start_progress(progress, len(target_images), skipping, log) - - # if pulling and save is True. Save every pulled image to assure parity - error_count = base.run_concurrent(workers, progress, download_docker_image, missing_images['not_pulled'], - save, output_dir, docker_client) - # only save those that are pulled already but not saved - error_count += base.run_concurrent(workers, progress, save_image, - missing_images['not_saved'] - missing_images['not_pulled'], - None, output_dir, docker_client) - - base.finish_progress(progress, error_count, log) - if error_count > 0: - log.error('{} images were not downloaded'.format(error_count)) - missing_images = missing(docker_client, target_images, save, output_dir) - log.info(check_table(merge_dict_sets(missing_images), missing_images, save)) - raise RuntimeError() - - -def run_cli(): - parser = argparse.ArgumentParser(description='Download docker images from list') - parser.add_argument('image_list', metavar='image-list', - help='File with list of images to download.') - parser.add_argument('--save', '-s', action='store_true', default=False, - help='Save images (without it only pull is executed)') - parser.add_argument('--output-dir', '-o', default=os.getcwd(), - help='Download destination') - parser.add_argument('--check', '-c', action='store_true', default=False, - help='Check what is missing. No download.' - 'Use with combination with -s to check saved images as well.') - parser.add_argument('--debug', action='store_true', default=False, - help='Turn on debug output') - parser.add_argument('--workers', type=int, default=3, - help='Set maximum workers for parallel download (default: 3)') - - args = parser.parse_args() - - if args.debug: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - else: - logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') - - progress = base.init_progress('Docker images') if not args.check else None - try: - download(args.image_list, args.save, args.output_dir, args.check, - progress, args.workers) - except docker.errors.DockerException: - log.exception('Irrecoverable error detected.') - sys.exit(1) - except RuntimeError as err: - log.exception(err) - - -if __name__ == '__main__': - run_cli() - |