From b6208f8299180f48144738edfead1cc8139925dd Mon Sep 17 00:00:00 2001 From: Bartek Grzybowski Date: Fri, 21 Jan 2022 11:20:49 +0100 Subject: [BUILD] Support docker mirror in download script A new option '--docker-private-registry-mirror' is added that allows pulling images which originate from private registries from custom docker mirroring repository (pull through cache) Change-Id: I28ee693438d3684d3d0000a7dd94a265c0a617c1 Issue-ID: OOM-2915 Signed-off-by: Bartek Grzybowski --- build/download/docker_downloader.py | 30 +++++++++++++++++++++++++++--- build/download/download.py | 7 +++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/build/download/docker_downloader.py b/build/download/docker_downloader.py index c5f53cab..af9d513d 100755 --- a/build/download/docker_downloader.py +++ b/build/download/docker_downloader.py @@ -36,8 +36,12 @@ log = logging.getLogger(__name__) class DockerDownloader(ConcurrentDownloader): - def __init__(self, save, *list_args, workers=3): + def __init__(self, save, *list_args, mirror=None, workers=3): + """ + :param mirror: private repository mirror address (ip:port) + """ self._save = save + self._mirror = mirror try: # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB) self._docker_client = docker.from_env(timeout=300) @@ -153,7 +157,24 @@ class DockerDownloader(ConcurrentDownloader): if ':' not in image_name.rsplit('/')[-1]: image_name = '{}:latest'.format(image_name) try: - image = self._docker_client.images.pull(image_name) + if self._mirror: + # if docker mirroring repository is set + image_name_split = image_name.split('/') + if (len(image_name_split) > 1) \ + and (image_name_split[0].find(".")) \ + and not (image_name.startswith('docker.io/')): + # if image originates from private registry and its name does not start with 'docker.io' + # download image from docker mirror and retag it to its original name + mirrored_image_name = self._mirror + "/" + '/'.join(image_name_split[1:]) + img = self._docker_client.images.pull(mirrored_image_name) + self._docker_client.images.model.tag(img, image_name) + # untag the image pulled from mirror + self._docker_client.images.remove(mirrored_image_name) + image = self._docker_client.images.get(image_name) + else: + image = self._docker_client.images.pull(image_name) + else: + image = self._docker_client.images.pull(image_name) log.info('Image {} pulled'.format(image_name)) return image except docker.errors.APIError as err: @@ -205,6 +226,9 @@ def run_cli(): help='Save images (without it only pull is executed)') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') + parser.add_argument('--private-registry-mirror', default=None, metavar='IP:PORT', + help='Address of docker mirroring repository that caches images' + ' from private registries to get those images from') parser.add_argument('--check', '-c', action='store_true', default=False, help='Check what is missing. No download.' 'Use with combination with -s to check saved images as well.') @@ -220,7 +244,7 @@ def run_cli(): else: logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') - downloader = DockerDownloader(args.save, [args.image_list, args.output_dir], workers=args.workers) + downloader = DockerDownloader(args.save, [args.image_list, args.output_dir], mirror=args.private_registry_mirror, workers=args.workers) if args.check: log.info('Check mode. No download will be executed.') diff --git a/build/download/download.py b/build/download/download.py index 6d76b369..d78c7ada 100755 --- a/build/download/download.py +++ b/build/download/download.py @@ -3,7 +3,7 @@ # COPYRIGHT NOTICE STARTS HERE -# Copyright 2019 © Samsung Electronics Co., Ltd. +# Copyright 2022 © Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -67,6 +67,9 @@ def parse_args(): help='pypi packages type list and directory to save downloaded files') parser.add_argument('--npm-registry', default='https://registry.npmjs.org', help='npm registry to use (default: https://registry.npmjs.org)') + parser.add_argument('--docker-private-registry-mirror', default=None, metavar='IP:PORT', + help='Address of docker mirroring repository that caches images' + ' from private registries to get those images from') parser.add_argument('--check', '-c', action='store_true', default=False, help='Check what is missing. No download.') parser.add_argument('--debug', action='store_true', default=False, @@ -173,7 +176,7 @@ def run_cli(): if args.docker: save = True if len(list(filter(lambda x: len(x) == 2, args.docker))) == len(args.docker) else False - docker = docker_downloader.DockerDownloader(save, *args.docker, workers=3) + docker = docker_downloader.DockerDownloader(save, *args.docker, mirror=args.docker_private_registry_mirror, workers=3) interval_start = handle_download(docker, args.check, errorred_lists, interval_start) if args.http: -- cgit 1.2.3-korg