diff options
Diffstat (limited to 'build/download')
-rwxr-xr-x | build/download/download-bin-tools.sh | 60 | ||||
-rwxr-xr-x | build/download/download-pip.sh | 48 | ||||
-rwxr-xr-x | build/download/download.py | 17 | ||||
-rwxr-xr-x | build/download/git_repos.py | 7 | ||||
-rwxr-xr-x | build/download/pypi_packages.py | 88 |
5 files changed, 109 insertions, 111 deletions
diff --git a/build/download/download-bin-tools.sh b/build/download/download-bin-tools.sh deleted file mode 100755 index 327e210f..00000000 --- a/build/download/download-bin-tools.sh +++ /dev/null @@ -1,60 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -usage () { - echo "Usage:" - echo -e "./$(basename $0) [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) ./downloads" -} - -if [ "${1}" == "-h" ] || [ -z "${1}" ] ; then - usage - exit 1 -else - OUTDIR="${1}" -fi - -# we are keeping just dublin support in dublin branch -KUBECTL_VERSION=${KUBECTL_VERSION:-1.13.5} -HELM_VERSION=${HELM_VERSION:-2.12.3} -RKE_VERSION=${RKE_VERSION:-0.2.1} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" - -download() { - url="$1" - url_file="${url%%\?*}" - file=$(basename "$url_file") - echo "Downloading $url" - curl -s --retry 5 -y 10 -Y 10 --location "$url" -o "$file" -} - -download "https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - -download "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" -tar -xf ./helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64/helm -O > helm -rm -f ./helm-v${HELM_VERSION}-linux-amd64.tar.gz - -download "https://github.com/rancher/rke/releases/download/v${RKE_VERSION}/rke_linux-amd64" -mv rke_linux-amd64 rke - -chmod a+x ./helm ./kubectl ./rke - -exit 0 diff --git a/build/download/download-pip.sh b/build/download/download-pip.sh deleted file mode 100755 index 7ab4b0c8..00000000 --- a/build/download/download-pip.sh +++ /dev/null @@ -1,48 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="$1" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi -LIST_FILE=$(readlink -f "$LIST_FILE") - - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -cd "$outdir" -for line in $(clean_list "$LIST_FILE"); do - echo "Downloading $cnt / $lines: $line" - pip download $line - cnt=$((cnt+1)) -done diff --git a/build/download/download.py b/build/download/download.py index ebce931a..0d8912ee 100755 --- a/build/download/download.py +++ b/build/download/download.py @@ -30,6 +30,7 @@ import docker_images import git_repos import http_files import npm_packages +import pypi_packages import rpm_packages log = logging.getLogger(name=__name__) @@ -54,6 +55,9 @@ def parse_args(): list_group.add_argument('--git', action='append', nargs=2, default=[], metavar=('list', 'dir-name'), help='git repo type list and directory to save downloaded files') + list_group.add_argument('--pypi', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='pypi packages type list and directory to save downloaded files') parser.add_argument('--npm-registry', default='https://registry.npmjs.org', help='npm registry to use (default: https://registry.npmjs.org)') parser.add_argument('--check', '-c', action='store_true', default=False, @@ -63,7 +67,7 @@ def parse_args(): args = parser.parse_args() - for arg in ('docker', 'npm', 'http', 'rpm', 'git'): + for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'): if getattr(args, arg): return args @@ -143,6 +147,17 @@ def run_cli(): except RuntimeError: list_with_errors.append(git_list[0]) + for pypi_list in args.pypi: + if args.check: + log.info('Check mode for pypi packages is not implemented') + break + progress = None if args.check else base.init_progress('pypi packages') + log.info('Processing {}.'.format(pypi_list[0])) + try: + pypi_packages.download(pypi_list[0], pypi_list[1], progress) + except RuntimeError: + list_with_errors.append(pypi_list[0]) + e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) log.info(timeit.default_timer() - timer_start) log.info('Execution ended. Total elapsed time {}'.format(e_time)) diff --git a/build/download/git_repos.py b/build/download/git_repos.py index aff01b80..1d8c2979 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -21,6 +21,7 @@ import argparse import subprocess +import shutil import logging import sys import os @@ -63,18 +64,20 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: + if os.path.isdir(dst): + shutil.rmtree(dst) log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - raise RuntimeError('Download unsuccesfull') + raise RuntimeError('Download unsuccessful') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') parser.add_argument('git_list', metavar='git-list', - help='File with list of npm packages to download.') + help='File with list of git repos to download.') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') diff --git a/build/download/pypi_packages.py b/build/download/pypi_packages.py new file mode 100755 index 00000000..951003c5 --- /dev/null +++ b/build/download/pypi_packages.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import subprocess +import os +from retrying import retry + +import base + +log = logging.getLogger(name=__name__) + +@retry(stop_max_attempt_number=5, wait_fixed=5000) +def download_package(package_name, dst_dir): + command = 'pip download --dest {} {}'.format(dst_dir, package_name) + log.info('Running: {}'.format(command)) + log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode()) + log.info('Downloaded: {}'.format(package_name)) + + +def download(pypi_list, dst_dir, progress): + if not base.check_tool('pip'): + log.error('ERROR: pip is not installed') + progress.finish(dirty=True) + raise RuntimeError('pip missing') + + pypi_set = base.load_list(pypi_list) + + error_count = 0 + + base.start_progress(progress, len(pypi_set), [], log) + + for package in pypi_set: + try: + download_package(package, dst_dir) + except subprocess.CalledProcessError as err: + log.exception(err.output.decode()) + error_count += 1 + + progress.update(progress.value + 1) + + base.finish_progress(progress, error_count, log) + if error_count > 0: + log.error('{} packages were not downloaded. Check logs for details'.format(error_count)) + raise RuntimeError('Download unsuccesfull') + + +def run_cli(): + parser = argparse.ArgumentParser(description='Download git repositories from list') + parser.add_argument('pypi_list', metavar='pypi-list', + help='File with list of pypi packages to download.') + parser.add_argument('--output-dir', '-o', default=os.getcwd(), + help='Download destination') + + args = parser.parse_args() + + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') + + progress = base.init_progress('pypi packages') + try: + download(args.pypi_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) + + +if __name__ == '__main__': + run_cli() |