summaryrefslogtreecommitdiffstats
path: root/build/download/base.py
diff options
context:
space:
mode:
Diffstat (limited to 'build/download/base.py')
-rw-r--r--build/download/base.py83
1 files changed, 83 insertions, 0 deletions
diff --git a/build/download/base.py b/build/download/base.py
new file mode 100644
index 00000000..fcf60242
--- /dev/null
+++ b/build/download/base.py
@@ -0,0 +1,83 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# COPYRIGHT NOTICE STARTS HERE
+
+# Copyright 2019 © Samsung Electronics Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# COPYRIGHT NOTICE ENDS HERE
+
+
+import progressbar
+import concurrent.futures
+
+progressbar.streams.wrap_stdout()
+progressbar.streams.wrap_stderr()
+
+
+def load_list(item_list):
+ """
+ Parse list with items to be downloaded.
+ :param item_list: File with list of items (1 line per item)
+ :return: set of items from file
+ """
+ with open(item_list, 'r') as f:
+ return {item for item in (line.strip() for line in f) if item}
+
+
+def init_progress(items_name):
+ progress_widgets = ['Downloading {}: '.format(items_name),
+ progressbar.Bar(), ' ',
+ progressbar.Percentage(), ' ',
+ '(', progressbar.SimpleProgress(), ')']
+
+ progress = progressbar.ProgressBar(widgets=progress_widgets,
+ poll_rate=1.0,
+ redirect_stdout=True)
+ return progress
+
+
+def start_progress(progress, target_count, skipping, log):
+ log_skipping(skipping, log)
+ log.info("Initializing download. Takes a while.")
+
+ progress.max_value = target_count
+ progress.start()
+ progress.update(len(skipping))
+
+
+def log_skipping(skipping_iterable, logger):
+ for skipped in skipping_iterable:
+ logger.info('Skipping: {}'.format(skipped))
+
+
+def run_concurrent(workers, progress, fn, iterable, *args):
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
+ futures = [executor.submit(fn, item, *args) for item in iterable]
+ error_count = 0
+ for future in concurrent.futures.as_completed(futures):
+ error = future.exception()
+ if error:
+ error_count += 1
+ progress.update()
+ else:
+ progress.update(progress.value +1)
+ return error_count
+
+
+def finish_progress(progress, error_count, log):
+ progress.finish(dirty=error_count > 0)
+ log.info('Download ended. Elapsed time {}'.format(progress.data()['time_elapsed']))
+