diff options
| -rwxr-xr-x | build/download/npm_packages.py | 121 |
1 files changed, 121 insertions, 0 deletions
diff --git a/build/download/npm_packages.py b/build/download/npm_packages.py new file mode 100755 index 00000000..c174e2c1 --- /dev/null +++ b/build/download/npm_packages.py @@ -0,0 +1,121 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import concurrent.futures +import hashlib +import logging +import os +import sys +from retrying import retry + +import base + +log = logging.getLogger(name=__name__) + + +@retry(stop_max_attempt_number=5, wait_fixed=5000) +def get_npm(registry, npm_name, npm_version): + npm_url = '{}/{}/{}'.format(registry, npm_name, npm_version) + npm_req = base.make_get_request(npm_url) + npm_json = npm_req.json() + tarball_url = npm_json['dist']['tarball'] + shasum = npm_json['dist']['shasum'] + tarball_req = base.make_get_request(tarball_url) + tarball = tarball_req.content + if hashlib.sha1(tarball).hexdigest() == shasum: + return tarball + else: + raise Exception('{}@{}: Wrong checksum. Retrying...'.format(npm_name, npm_version)) + + +def download_npm(npm, registry, dst_dir): + log.info('Downloading: {}'.format(npm)) + npm_name, npm_version = npm.split('@') + dst_path = '{}/{}-{}.tgz'.format(dst_dir, npm_name, npm_version) + try: + tarball = get_npm(registry, *npm.split('@')) + base.save_to_file(dst_path, tarball) + except Exception as err: + if os.path.isfile(dst_path): + os.remove(dst_path) + log.error('Failed: {}: {}'.format(npm, err)) + raise err + log.info('Downloaded: {}'.format(npm)) + + +def missing(npm_set, dst_dir): + return {npm for npm in npm_set + if not os.path.isfile('{}/{}-{}.tgz'.format(dst_dir, *npm.split('@')))} + + +def download(npm_list, registry, dst_dir, check_mode, progress=None, workers=None): + npm_set = base.load_list(npm_list) + target_count = len(npm_set) + missing_npms = missing(npm_set, dst_dir) + + if check_mode: + log.info(base.simple_check_table(npm_set, missing_npms)) + return 0 + + skipping = npm_set - missing_npms + + base.start_progress(progress, len(npm_set), skipping, log) + error_count = base.run_concurrent(workers, progress, download_npm, missing_npms, registry, dst_dir) + + if error_count > 0: + log.error('{} packages were not downloaded. Check log for specific failures.'.format(error_count)) + + base.finish_progress(progress, error_count, log) + + return error_count + + +def run_cli(): + parser = argparse.ArgumentParser(description='Download npm packages from list') + parser.add_argument('npm_list', metavar='npm-list', + help='File with list of npm packages to download.') + parser.add_argument('--registry', '-r', default='https://registry.npmjs.org', + help='Download destination') + parser.add_argument('--output-dir', '-o', default=os.getcwd(), + help='Download destination') + parser.add_argument('--check', '-c', action='store_true', default=False, + help='Check what is missing. No download.') + parser.add_argument('--debug', action='store_true', default=False, + help='Turn on debug output') + parser.add_argument('--workers', type=int, default=None, + help='Set maximum workers for parallel download (default: cores * 5)') + + args = parser.parse_args() + + if args.debug: + logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + else: + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') + + progress = base.init_progress('npm packages') if not args.check else None + sys.exit(download(args.npm_list, args.registry, args.output_dir, args.check, progress, + args.workers)) + + +if __name__ == '__main__': + run_cli() + |