Improve docker image download script

This is part of the series of changes that will improve
download scripts.

Issue-ID: OOM-1803

Change-Id: I6606762f8e05c25132257e5612997418a9d6d701
Signed-off-by: Milan Verespej <m.verespej@partner.samsung.com>

1 files changed, 83 insertions, 0 deletions

diff --git a/build/download/base.py b/build/download/base.py
new file mode 100644
index 00000000..fcf60242
--- /dev/null
+++ b/build/download/base.py
@@ -0,0 +1,83 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+#   COPYRIGHT NOTICE STARTS HERE
+
+#   Copyright 2019 © Samsung Electronics Co., Ltd.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+#   COPYRIGHT NOTICE ENDS HERE
+
+
+import progressbar
+import concurrent.futures
+
+progressbar.streams.wrap_stdout()
+progressbar.streams.wrap_stderr()
+
+
+def load_list(item_list):
+    """
+    Parse list with items to be downloaded.
+    :param item_list: File with list of items (1 line per item)
+    :return: set of items from file
+    """
+    with open(item_list, 'r') as f:
+        return {item for item in (line.strip() for line in f) if item}
+
+
+def init_progress(items_name):
+    progress_widgets = ['Downloading {}: '.format(items_name),
+                        progressbar.Bar(), ' ',
+                        progressbar.Percentage(), ' ',
+                        '(', progressbar.SimpleProgress(), ')']
+
+    progress = progressbar.ProgressBar(widgets=progress_widgets,
+                                       poll_rate=1.0,
+                                       redirect_stdout=True)
+    return progress
+
+
+def start_progress(progress, target_count, skipping, log):
+    log_skipping(skipping, log)
+    log.info("Initializing download. Takes a while.")
+
+    progress.max_value = target_count
+    progress.start()
+    progress.update(len(skipping))
+
+
+def log_skipping(skipping_iterable, logger):
+    for skipped in skipping_iterable:
+        logger.info('Skipping: {}'.format(skipped))
+
+
+def run_concurrent(workers, progress, fn, iterable, *args):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+        futures = [executor.submit(fn, item, *args) for item in iterable]
+        error_count = 0
+        for future in concurrent.futures.as_completed(futures):
+            error = future.exception()
+            if error:
+                error_count += 1
+                progress.update()
+            else:
+                progress.update(progress.value +1)
+    return error_count
+
+
+def finish_progress(progress, error_count, log):
+    progress.finish(dirty=error_count > 0)
+    log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
+