diff options
Diffstat (limited to 'build')
22 files changed, 331 insertions, 617 deletions
diff --git a/build/creating_data/create-rhel-repo.sh b/build/creating_data/create-rhel-repo.sh deleted file mode 100755 index 43709a7e..00000000 --- a/build/creating_data/create-rhel-repo.sh +++ /dev/null @@ -1,45 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -OUTDIR="${1}" -if [[ -z "${OUTDIR}" ]]; then - echo "Missing output dir" - exit 1 -fi - -# if onap.repo does not exists create it -mkdir -p "${OUTDIR}" -if [ ! -f "${OUTDIR}/onap.repo" ]; then - cat > "${OUTDIR}/onap.repo" <<EOF -[ONAP] -name=Offline ONAP repository -baseurl=PATH -enabled=1 -gpgcheck=0 -EOF -fi - -# this exact docker version is required by ONAP/beijing -# it should be available in centos docker repo -yumdownloader --resolve --destdir="${OUTDIR}" docker-ce-18.09.5 container-selinux docker-ce-cli \ -containerd.io nfs-utils python-jsonpointer python-docker-py python-docker-pycreds python-ipaddress \ -python-websocket-client - -createrepo "${OUTDIR}" - -exit 0
\ No newline at end of file diff --git a/build/creating_data/create-ubuntu-repo.sh b/build/creating_data/create-ubuntu-repo.sh deleted file mode 100755 index ac7de65c..00000000 --- a/build/creating_data/create-ubuntu-repo.sh +++ /dev/null @@ -1,33 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -OUTDIR="${1}" -if [[ -z "${OUTDIR}" ]]; then - echo "Missing output dir" - exit 1 -fi - - -# create the package index -dpkg-scanpackages -m "${OUTDIR}" > "${OUTDIR}/Packages" -cat "${OUTDIR}/Packages" | gzip -9c > "${OUTDIR}/Packages.gz" - -# create the Release file -echo 'deb [trusted=yes] http://repo.infra-server/ubuntu/xenial /' > "${OUTDIR}/onap.list" - -exit 0 diff --git a/build/creating_data/download-bin-tools.sh b/build/creating_data/download-bin-tools.sh deleted file mode 100755 index 327e210f..00000000 --- a/build/creating_data/download-bin-tools.sh +++ /dev/null @@ -1,60 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -usage () { - echo "Usage:" - echo -e "./$(basename $0) [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) ./downloads" -} - -if [ "${1}" == "-h" ] || [ -z "${1}" ] ; then - usage - exit 1 -else - OUTDIR="${1}" -fi - -# we are keeping just dublin support in dublin branch -KUBECTL_VERSION=${KUBECTL_VERSION:-1.13.5} -HELM_VERSION=${HELM_VERSION:-2.12.3} -RKE_VERSION=${RKE_VERSION:-0.2.1} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" - -download() { - url="$1" - url_file="${url%%\?*}" - file=$(basename "$url_file") - echo "Downloading $url" - curl -s --retry 5 -y 10 -Y 10 --location "$url" -o "$file" -} - -download "https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - -download "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" -tar -xf ./helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64/helm -O > helm -rm -f ./helm-v${HELM_VERSION}-linux-amd64.tar.gz - -download "https://github.com/rancher/rke/releases/download/v${RKE_VERSION}/rke_linux-amd64" -mv rke_linux-amd64 rke - -chmod a+x ./helm ./kubectl ./rke - -exit 0 diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh deleted file mode 100755 index c0a0bed1..00000000 --- a/build/creating_data/download-docker-images.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - LIST_FILE="docker_image_list.txt" -fi - -echo "Download all images" - -lines=$(clean_list "$LIST_FILE" | wc -l) -line=1 -for image in $(clean_list "$LIST_FILE"); do - echo "== pkg #$line of $lines ==" - echo "$image" - retry docker -l error pull "$image" - line=$((line+1)) -done diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh deleted file mode 100755 index f687fda1..00000000 --- a/build/creating_data/download-files.sh +++ /dev/null @@ -1,50 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -for line in $(clean_list "$LIST_FILE"); do - # www.springframework.org/schema/tool/spring-tool-4.3.xsd - file="${line%%\?*}" - filename=$(basename "$file") - echo "Downloading $cnt / $lines: $file" - # following curl params are ensurring 5 reties and cut-off if connectivity will - # drop below 10b/10s - curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$filename" &>/dev/null - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-git-repos.sh b/build/creating_data/download-git-repos.sh deleted file mode 100755 index 7853a141..00000000 --- a/build/creating_data/download-git-repos.sh +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/env bash -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -# fail fast -set -e - -usage () { - echo "Usage:" - echo -e "./$(basename $0) <repository list> [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) onap_3.0.x-git_repos.list ./git-repo" -} - -LIST="${1}" - -if [[ -z "${LIST}" ]]; then - echo "Missing argument for repository list" - exit 1 -fi - -OUTDIR="${2}" -if [[ -z "${OUTDIR}" ]]; then - OUTDIR="./git-repo" -fi - -mkdir -p "${OUTDIR}" -cd "${OUTDIR}" - - -while IFS=" " read -r REPO BRANCH remainder -do - if [[ -z "${BRANCH}" ]]; then - git clone https://${REPO} --bare ${REPO} - else - git clone -b ${BRANCH} --single-branch https://${REPO} --bare ${REPO} - fi -done < <(awk '$1 ~ /^[^;#]/' ${LIST}) - - -exit 0 diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh deleted file mode 100755 index 1144c66e..00000000 --- a/build/creating_data/download-http-files.sh +++ /dev/null @@ -1,51 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -for line in $(clean_list "$LIST_FILE"); do - # www.springframework.org/schema/tool/spring-tool-4.3.xsd - file="${line%%\?*}" - echo "Downloading $cnt / $lines: $file" - fdir=$(dirname "$file") - mkdir -p $outdir/$fdir - # following curl params are ensurring 5 reties and cut-off if connectivity will - # drop below 10b/10s - curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh deleted file mode 100755 index 191dd5df..00000000 --- a/build/creating_data/download-npm-pkgs.sh +++ /dev/null @@ -1,42 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" - -if [[ -z "$LIST_FILE" ]]; then - LIST_FILE="all_npm_list.txt" -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing arg outdir" - exit 1 -fi - -mkdir -p "$outdir" -cd "$outdir" -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 -for line in $(clean_list "$LIST_FILE"); do - echo "== pkg #$cnt of $lines ==" - npm pack $line - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-pip.sh b/build/creating_data/download-pip.sh deleted file mode 100755 index 7ab4b0c8..00000000 --- a/build/creating_data/download-pip.sh +++ /dev/null @@ -1,48 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="$1" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi -LIST_FILE=$(readlink -f "$LIST_FILE") - - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -cd "$outdir" -for line in $(clean_list "$LIST_FILE"); do - echo "Downloading $cnt / $lines: $line" - pip download $line - cnt=$((cnt+1)) -done diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh deleted file mode 100755 index 0a72d15b..00000000 --- a/build/creating_data/save-docker-images.sh +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -IMG_DIR="${2}" - -if [[ -z "$IMG_DIR" ]]; then - IMG_DIR="./images" -fi - -echo "Creating ${IMG_DIR}" -if [[ ! -d "${IMG_DIR}" ]]; then - mkdir -p "${IMG_DIR}" -fi - -save_image() { - local name_tag=$1 - echo "$name_tag" - local img_name=$(echo "${name_tag}" | tr /: __) - local img_path="${IMG_DIR}/${img_name}.tar" - - if [[ ! -f "${img_path}" ]] ; then - echo "[DEBUG] save ${name_tag} to ${img_path}" - echo "${name_tag}" >> $IMG_DIR/_image_list.txt - retry docker -l error save -o "${img_path}" ${name_tag} - else - echo "[DEBUG] ${name_tag} already saved" - fi -} - -echo "Save all images" -line=1 -lines=$(clean_list "$LIST_FILE" | wc -l) -for image in $(clean_list "$LIST_FILE"); do - echo "== pkg #$line of $lines ==" - save_image "${image}" - line=$((line+1)) -done
\ No newline at end of file diff --git a/build/data_lists/infra_bin_utils.list b/build/data_lists/infra_bin_utils.list new file mode 100644 index 00000000..8de616e9 --- /dev/null +++ b/build/data_lists/infra_bin_utils.list @@ -0,0 +1,3 @@ +https://storage.googleapis.com/kubernetes-release/release/v1.13.5/bin/linux/amd64/kubectl +https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz +https://github.com/rancher/rke/releases/download/v0.2.1/rke_linux-amd64 diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list index ab4949b2..48029bc2 100644 --- a/build/data_lists/onap_pip_packages.list +++ b/build/data_lists/onap_pip_packages.list @@ -5,7 +5,7 @@ Flask==1.0.3 idna==2.8 itsdangerous==1.1.0 Jinja2==2.10.1 -MarkupSafe==1.1.1 +MarkupSafe==0.23 requests==2.22.0 urllib3==1.25.3 Werkzeug==0.15.4 diff --git a/build/data_lists/onap_rpm.list b/build/data_lists/onap_rpm.list new file mode 100644 index 00000000..4595d4b1 --- /dev/null +++ b/build/data_lists/onap_rpm.list @@ -0,0 +1,21 @@ +containerd.io-1.2.5-3.1.el7.x86_64 +container-selinux-1.12.5-14.el7.x86_64 +container-selinux-2.95-2.el7_6.noarch +docker-ce-18.09.5-3.el7.x86_64 +docker-ce-cli-18.09.6-3.el7.x86_64 +gssproxy-0.7.0-21.el7.x86_64 +keyutils-1.5.8-3.el7.x86_64 +libbasicobjects-0.1.1-32.el7.x86_64 +libcollection-0.7.0-32.el7.x86_64 +libevent-2.0.21-4.el7.x86_64 +libini_config-1.3.1-32.el7.x86_64 +libnfsidmap-0.25-19.el7.x86_64 +libpath_utils-0.2.1-32.el7.x86_64 +libref_array-0.1.5-32.el7.x86_64 +libverto-libevent-0.2.5-4.el7.x86_64 +nfs-utils-1.3.0-0.61.el7.x86_64 +python-docker-py-1.10.6-9.el7_6.noarch +python-docker-pycreds-0.3.0-9.el7_6.noarch +python-ipaddress-1.0.16-2.el7.noarch +python-jsonpointer-1.9-2.el7.noarch +python-websocket-client-0.32.0-116.el7.noarch diff --git a/build/download/base.py b/build/download/base.py index 5bcd0ef6..d8b44839 100644 --- a/build/download/base.py +++ b/build/download/base.py @@ -38,7 +38,8 @@ def load_list(item_list): :return: set of items from file """ with open(item_list, 'r') as f: - return {item for item in (line.strip() for line in f) if item} + return {item for item in (line.strip() for line in f) + if item and not item.startswith('#')} def init_progress(items_name): diff --git a/build/download/docker_images.py b/build/download/docker_images.py index e4e742b3..d8138dd6 100755 --- a/build/download/docker_images.py +++ b/build/download/docker_images.py @@ -180,7 +180,7 @@ def download_docker_image(image, save, output_dir, docker_client): if save: save_image(image, pulled_image, output_dir) except Exception as err: - log.error('Error downloading {}: {}'.format(image, err)) + log.exception('Error downloading {}: {}'.format(image, err)) raise err @@ -195,10 +195,10 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): :return: None """ try: - docker_client = docker.client.DockerClient(version='auto') + # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB) + docker_client = docker.client.DockerClient(version='auto', timeout=300) except docker.errors.DockerException as err: - log.error(err) - log.error('Error creating docker client. Check if is docker installed and running' + log.exception('Error creating docker client. Check if is docker installed and running' ' or if you have right permissions.') raise err @@ -221,14 +221,12 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): missing_images['not_saved'] - missing_images['not_pulled'], None, output_dir, docker_client) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} images were not downloaded'.format(error_count)) missing_images = missing(docker_client, target_images, save, output_dir) log.info(check_table(merge_dict_sets(missing_images), missing_images, save)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): @@ -256,11 +254,13 @@ def run_cli(): progress = base.init_progress('Docker images') if not args.check else None try: - sys.exit(download(args.image_list, args.save, args.output_dir, args.check, - progress, args.workers)) + download(args.image_list, args.save, args.output_dir, args.check, + progress, args.workers) except docker.errors.DockerException: - log.error('Irrecoverable error detected.') + log.exception('Irrecoverable error detected.') sys.exit(1) + except RuntimeError as err: + log.exception(err) if __name__ == '__main__': diff --git a/build/download/download.py b/build/download/download.py new file mode 100755 index 00000000..0d8912ee --- /dev/null +++ b/build/download/download.py @@ -0,0 +1,173 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import datetime +import timeit + +import base +import docker_images +import git_repos +import http_files +import npm_packages +import pypi_packages +import rpm_packages + +log = logging.getLogger(name=__name__) + +def parse_args(): + parser=argparse.ArgumentParser(description='Download data from lists') + list_group = parser.add_argument_group() + list_group.add_argument('--docker', action='append', nargs='+', default=[], + metavar=('list', 'dir-name'), + help='Docker type list. If second argument is specified ' + 'it is treated as directory where images will be saved ' + 'otherwise only pull operation is executed') + list_group.add_argument('--http', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='Http type list and directory to save downloaded files') + list_group.add_argument('--npm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='npm type list and directory to save downloaded files') + list_group.add_argument('--rpm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='rpm type list and directory to save downloaded files') + list_group.add_argument('--git', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='git repo type list and directory to save downloaded files') + list_group.add_argument('--pypi', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='pypi packages type list and directory to save downloaded files') + parser.add_argument('--npm-registry', default='https://registry.npmjs.org', + help='npm registry to use (default: https://registry.npmjs.org)') + parser.add_argument('--check', '-c', action='store_true', default=False, + help='Check what is missing. No download.') + parser.add_argument('--debug', action='store_true', default=False, + help='Turn on debug output') + + args = parser.parse_args() + + for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'): + if getattr(args, arg): + return args + + parser.error('One of --docker, --npm, --http, --rpm, --git must be specified') + + +def run_cli(): + args = parse_args() + + console_handler = logging.StreamHandler(sys.stdout) + console_formatter = logging.Formatter('%(message)s') + console_handler.setFormatter(console_formatter) + now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + log_file = 'download_data-{}.log'.format(now) + file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s" + + if args.debug: + logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format) + else: + logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format) + root_logger = logging.getLogger() + root_logger.addHandler(console_handler) + + list_with_errors = [] + timer_start = timeit.default_timer() + + for docker_list in args.docker: + log.info('Processing {}.'.format(docker_list[0])) + progress = None if args.check else base.init_progress('docker images') + save = False + if len(docker_list) > 1: + save = True + else: + docker_list.append(None) + try: + docker_images.download(docker_list[0], save, + docker_list[1], args.check, progress) + except RuntimeError: + list_with_errors.append(docker_list[0]) + + for http_list in args.http: + progress = None if args.check else base.init_progress('http files') + log.info('Processing {}.'.format(http_list[0])) + try: + http_files.download(http_list[0], http_list[1], args.check, + progress) + except RuntimeError: + list_with_errors.append(http_list[0]) + + for npm_list in args.npm: + progress = None if args.check else base.init_progress('npm packages') + log.info('Processing {}.'.format(npm_list[0])) + try: + npm_packages.download(npm_list[0], args.npm_registry, npm_list[1], + args.check, progress) + except RuntimeError: + list_with_errors.append(npm_list[0]) + + for rpm_list in args.rpm: + if args.check: + log.info('Check mode for rpm packages is not implemented') + break + log.info('Processing {}.'.format(rpm_list[0])) + try: + rpm_packages.download(rpm_list[0], rpm_list[1]) + except RuntimeError: + list_with_errors.append(rpm_list[0]) + + for git_list in args.git: + if args.check: + log.info('Check mode for git repositories is not implemented') + break + progress = None if args.check else base.init_progress('git repositories') + log.info('Processing {}.'.format(git_list[0])) + try: + git_repos.download(git_list[0], git_list[1], progress) + except RuntimeError: + list_with_errors.append(git_list[0]) + + for pypi_list in args.pypi: + if args.check: + log.info('Check mode for pypi packages is not implemented') + break + progress = None if args.check else base.init_progress('pypi packages') + log.info('Processing {}.'.format(pypi_list[0])) + try: + pypi_packages.download(pypi_list[0], pypi_list[1], progress) + except RuntimeError: + list_with_errors.append(pypi_list[0]) + + e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) + log.info(timeit.default_timer() - timer_start) + log.info('Execution ended. Total elapsed time {}'.format(e_time)) + + if list_with_errors: + log.error('Errors encountered while processing these lists:' + '\n{}'.format('\n'.join(list_with_errors))) + sys.exit(1) + + + +if __name__ == '__main__': + run_cli() diff --git a/build/download/git_repos.py b/build/download/git_repos.py index e388e94c..1d8c2979 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -21,6 +21,7 @@ import argparse import subprocess +import shutil import logging import sys import os @@ -45,10 +46,9 @@ def download(git_list, dst_dir, progress): if not base.check_tool('git'): log.error('ERROR: git is not installed') progress.finish(dirty=True) - return 1 + raise RuntimeError('git missing') - git_set = {tuple(item.split()) for item in base.load_list(git_list) - if not item.startswith('#')} + git_set = {tuple(item.split()) for item in base.load_list(git_list)} error_count = 0 @@ -64,19 +64,20 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: - log.error(err.output.decode()) + if os.path.isdir(dst): + shutil.rmtree(dst) + log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - return error_count - + raise RuntimeError('Download unsuccessful') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') parser.add_argument('git_list', metavar='git-list', - help='File with list of npm packages to download.') + help='File with list of git repos to download.') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') @@ -85,8 +86,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') progress = base.init_progress('git repositories') - - sys.exit(download(args.git_list, args.output_dir, progress)) + try: + download(args.git_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/http_files.py b/build/download/http_files.py index f5b1e59a..c83158d6 100755 --- a/build/download/http_files.py +++ b/build/download/http_files.py @@ -83,7 +83,7 @@ def download(data_list, dst_dir, check, progress, workers=None): if check: log.info(base.simple_check_table(file_set, missing_files)) - return 0 + return skipping = file_set - missing_files @@ -91,12 +91,11 @@ def download(data_list, dst_dir, check, progress, workers=None): error_count = base.run_concurrent(workers, progress, download_file, missing_files, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} files were not downloaded. Check log for specific failures.'.format(error_count)) + raise RuntimeError() - base.finish_progress(progress, error_count, log) - - return error_count def run_cli(): """ @@ -123,7 +122,10 @@ def run_cli(): progress = base.init_progress('http files') if not args.check else None - sys.exit(download(args.file_list, args.output_dir, args.check, progress, args.workers)) + try: + download(args.file_list, args.output_dir, args.check, progress, args.workers) + except RuntimeError: + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/npm_packages.py b/build/download/npm_packages.py index c174e2c1..70c03ad8 100755 --- a/build/download/npm_packages.py +++ b/build/download/npm_packages.py @@ -57,7 +57,7 @@ def download_npm(npm, registry, dst_dir): except Exception as err: if os.path.isfile(dst_path): os.remove(dst_path) - log.error('Failed: {}: {}'.format(npm, err)) + log.exception('Failed: {}'.format(npm)) raise err log.info('Downloaded: {}'.format(npm)) @@ -81,12 +81,10 @@ def download(npm_list, registry, dst_dir, check_mode, progress=None, workers=Non base.start_progress(progress, len(npm_set), skipping, log) error_count = base.run_concurrent(workers, progress, download_npm, missing_npms, registry, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} packages were not downloaded. Check log for specific failures.'.format(error_count)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): diff --git a/build/download/pypi_packages.py b/build/download/pypi_packages.py new file mode 100755 index 00000000..951003c5 --- /dev/null +++ b/build/download/pypi_packages.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import subprocess +import os +from retrying import retry + +import base + +log = logging.getLogger(name=__name__) + +@retry(stop_max_attempt_number=5, wait_fixed=5000) +def download_package(package_name, dst_dir): + command = 'pip download --dest {} {}'.format(dst_dir, package_name) + log.info('Running: {}'.format(command)) + log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode()) + log.info('Downloaded: {}'.format(package_name)) + + +def download(pypi_list, dst_dir, progress): + if not base.check_tool('pip'): + log.error('ERROR: pip is not installed') + progress.finish(dirty=True) + raise RuntimeError('pip missing') + + pypi_set = base.load_list(pypi_list) + + error_count = 0 + + base.start_progress(progress, len(pypi_set), [], log) + + for package in pypi_set: + try: + download_package(package, dst_dir) + except subprocess.CalledProcessError as err: + log.exception(err.output.decode()) + error_count += 1 + + progress.update(progress.value + 1) + + base.finish_progress(progress, error_count, log) + if error_count > 0: + log.error('{} packages were not downloaded. Check logs for details'.format(error_count)) + raise RuntimeError('Download unsuccesfull') + + +def run_cli(): + parser = argparse.ArgumentParser(description='Download git repositories from list') + parser.add_argument('pypi_list', metavar='pypi-list', + help='File with list of pypi packages to download.') + parser.add_argument('--output-dir', '-o', default=os.getcwd(), + help='Download destination') + + args = parser.parse_args() + + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') + + progress = base.init_progress('pypi packages') + try: + download(args.pypi_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) + + +if __name__ == '__main__': + run_cli() diff --git a/build/download/rpm_packages.py b/build/download/rpm_packages.py index 7f9700a3..732af0ea 100755 --- a/build/download/rpm_packages.py +++ b/build/download/rpm_packages.py @@ -33,7 +33,7 @@ log = logging.getLogger(name=__name__) def download(rpm_list, dst_dir): if not base.check_tool('yumdownloader'): log.error('ERROR: yumdownloader is not installed') - return 1 + raise RuntimeError('yumdownloader missing') rpm_set = base.load_list(rpm_list) @@ -41,11 +41,10 @@ def download(rpm_list, dst_dir): log.info('Running command: {}'.format(command)) try: subprocess.check_call(command.split()) - log.info('Downloaded') except subprocess.CalledProcessError as err: - log.error(err.output) - return err.returncode - + log.exception(err.output) + raise err + log.info('Downloaded') def run_cli(): @@ -59,7 +58,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') - sys.exit(download(args.rpm_list, args.output_dir)) + try: + download(args.rpm_list, args.output_dir) + except (subprocess.CalledProcessError, RuntimeError): + sys.exit(1) + if __name__ == '__main__': diff --git a/build/download_offline_data_by_lists.sh b/build/download_offline_data_by_lists.sh deleted file mode 100755 index b2afd172..00000000 --- a/build/download_offline_data_by_lists.sh +++ /dev/null @@ -1,96 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# fail fast -set -e - -usage () { - echo "Usage:" - echo -e "./$(basename $0) <project version>\n" - echo "onap_3.0.0 for casablanca (sign-off 30/11/2018)" - echo "onap_3.0.1 for casablanca maintenance release (sign-off 10/12/2018)" - echo "onap_3.0.2 for latest casablanca with fixed certificates (sign-off 25/04/2019)" - echo "" - echo "Example:" - echo " ./$(basename $0) onap_3.0.2" -} - -# boilerplate -RELATIVE_PATH=./ # relative path from this script to 'common-functions.sh' -if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then - SCRIPT_DIR=$(dirname "${0}") - LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") - . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh -fi - -if [ "${1}" == "-h" ] || [ -z "${1}" ]; then - usage - exit 0 -else - TAG="${1}" -fi - -CTOOLS="${LOCAL_PATH}/creating_data" -LISTS_DIR="${LOCAL_PATH}/data_lists" -DATA_DIR="${LOCAL_PATH}/../../resources" -TOTAL=12 -CURR=1 - -message info "Downloading started: $(date)" - -echo "[Step $((CURR++))/$TOTAL Download collected docker images]" -$CTOOLS/download-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list" - -echo "[Step $((CURR++))/$TOTAL Download docker images for infra-server]" -$CTOOLS/download-docker-images.sh "${LISTS_DIR}/infra_docker_images.list" - -echo "[Step $((CURR++))/$TOTAL Build own nginx image]" -$CTOOLS/create_nginx_image/01create-image.sh "${DATA_DIR}/offline_data/docker_images_infra" - -echo "[Step $((CURR++))/$TOTAL Save docker images from docker cache to tarfiles]" -$CTOOLS/save-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list" "${DATA_DIR}/offline_data/docker_images_for_nexus" - -echo "[Step $((CURR++))/$TOTAL Prepare infra related images to infra folder]" -$CTOOLS/save-docker-images.sh "${LISTS_DIR}/infra_docker_images.list" "${DATA_DIR}/offline_data/docker_images_infra" - -echo "[Step $((CURR++))/$TOTAL Download git repos]" -$CTOOLS/download-git-repos.sh "${LISTS_DIR}/onap_3.0.x-git_repos.list" "${DATA_DIR}/git-repo" - -echo "[Step $((CURR++))/$TOTAL Download http files]" -$CTOOLS/download-http-files.sh "${LISTS_DIR}/onap_3.0.x-http_files.list" "${DATA_DIR}/http" - -echo "[Step $((CURR++))/$TOTAL Download npm pkgs]" -$CTOOLS/download-npm-pkgs.sh "${LISTS_DIR}/onap_3.0.x-npm.list" "${DATA_DIR}/offline_data/npm_tar" - -echo "[Step $((CURR++))/$TOTAL Download bin tools]" -$CTOOLS/download-bin-tools.sh "${DATA_DIR}/downloads" - -echo "[Step $((CURR++))/$TOTAL Create RHEL repository]" -$CTOOLS/create-rhel-repo.sh "${DATA_DIR}/pkg/rhel" - -echo "[Step $((CURR++))/$TOTAL Download sdnc-ansible-server packages]" -$CTOOLS/download-pip.sh "${LISTS_DIR}/onap_3.0.x-pip_packages.list" "${DATA_DIR}/offline_data/pypi" -$CTOOLS/download-files.sh "${LISTS_DIR}/deb_packages.list" "${DATA_DIR}/pkg/ubuntu/xenial" - -echo "[Step $((CURR++))/$TOTAL Create APT repository]" -$CTOOLS/create-ubuntu-repo.sh "${DATA_DIR}/pkg/ubuntu/xenial" - -message info "Downloading finished: $(date)" |