diff options
30 files changed, 290 insertions, 180 deletions
diff --git a/ansible/group_vars/infrastructure.yml b/ansible/group_vars/infrastructure.yml index 66a00b5f..c685e773 100755 --- a/ansible/group_vars/infrastructure.yml +++ b/ansible/group_vars/infrastructure.yml @@ -3,8 +3,6 @@ vnc_passwd: samsung simulated_hosts: git: - gerrit.onap.org - - git.rancher.io - - github.com http: - git.onap.org - nexus.onap.org diff --git a/ansible/roles/application/defaults/main.yml b/ansible/roles/application/defaults/main.yml index 84fffeca..2ae668ac 100644 --- a/ansible/roles/application/defaults/main.yml +++ b/ansible/roles/application/defaults/main.yml @@ -11,3 +11,5 @@ helm_extra_install_options: app_skip_helm_override: false app_helm_override_role: application-override app_helm_override_file: "{{ app_data_path }}/override.yaml" +helm_overide_files: + - "{{ app_helm_override_file }}" diff --git a/ansible/roles/application/tasks/install.yml b/ansible/roles/application/tasks/install.yml index bdf6e511..003631d7 100644 --- a/ansible/roles/application/tasks/install.yml +++ b/ansible/roles/application/tasks/install.yml @@ -71,7 +71,7 @@ {{ app_helm_release_name }} {{ helm_repository_name }}/{{ app_helm_chart_name }} --namespace {{ app_kubernetes_namespace }} - {{ '' if app_skip_helm_override else '-f ' + app_helm_override_file }} + {% if not app_skip_helm_override %} {% for arg in helm_overide_files %} {{ '-f ' + arg }} {% endfor %} {% endif %} {% for arg in helm_extra_install_options %} {{ arg.opt }} {% endfor %} changed_when: true # when executed its a changed type of action register: helm_install diff --git a/ansible/roles/helm/.yamllint b/ansible/roles/helm/.yamllint new file mode 100644 index 00000000..ad0be760 --- /dev/null +++ b/ansible/roles/helm/.yamllint @@ -0,0 +1,11 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable diff --git a/ansible/roles/helm/molecule/default/molecule.yml b/ansible/roles/helm/molecule/default/molecule.yml new file mode 100644 index 00000000..869f87f6 --- /dev/null +++ b/ansible/roles/helm/molecule/default/molecule.yml @@ -0,0 +1,32 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6} + pre_build_image: True + privileged: true + override_command: False + groups: + - infrastructure +provisioner: + name: ansible + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + inventory: + group_vars: + all: + app_name: onap + app_data_path: "/opt/{{ app_name }}" + helm_bin_dir: /usr/local/bin +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/helm/molecule/default/playbook.yml b/ansible/roles/helm/molecule/default/playbook.yml new file mode 100644 index 00000000..2705b165 --- /dev/null +++ b/ansible/roles/helm/molecule/default/playbook.yml @@ -0,0 +1,5 @@ +--- +- name: Converge + hosts: all + roles: + - helm diff --git a/ansible/roles/helm/molecule/default/prepare.yml b/ansible/roles/helm/molecule/default/prepare.yml new file mode 100644 index 00000000..8a149b89 --- /dev/null +++ b/ansible/roles/helm/molecule/default/prepare.yml @@ -0,0 +1,5 @@ +--- +- name: Prepare for helm tests + hosts: all + roles: + - prepare-helm diff --git a/ansible/roles/helm/molecule/default/tests/test_default.py b/ansible/roles/helm/molecule/default/tests/test_default.py new file mode 100644 index 00000000..2395183b --- /dev/null +++ b/ansible/roles/helm/molecule/default/tests/test_default.py @@ -0,0 +1,11 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_helm(host): + assert host.file('/usr/local/bin/helm').exists + assert host.run('helm').rc != 127 diff --git a/ansible/roles/nfs/molecule/default/molecule.yml b/ansible/roles/nfs/molecule/default/molecule.yml index 7bacf3c4..a8ca6a30 100644 --- a/ansible/roles/nfs/molecule/default/molecule.yml +++ b/ansible/roles/nfs/molecule/default/molecule.yml @@ -19,7 +19,7 @@ platforms: - name: nfs-net volumes: - /sys/fs/cgroup:/sys/fs/cgroup:ro - - ${HOME}/data:/dockerdata-nfs:rw + - /dockerdata-nfs - name: kubernetes-node-2 image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6} diff --git a/ansible/test/play-resources/molecule/default/cleanup.yml b/ansible/test/play-resources/molecule/default/cleanup.yml deleted file mode 100644 index e0c0b624..00000000 --- a/ansible/test/play-resources/molecule/default/cleanup.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Cleanup data from instance (doing it from hosts requires root access). - hosts: resources - gather_facts: false - ignore_unreachable: true - pre_tasks: - - name: Find files and dirs to delete - find: - paths: /data/ # Only deleting content not dir itself as we get "Device or resource busy" error as it's mounted to container doing the deletion - patterns: "*" - recurse: true - file_type: any - register: files_to_delete - - name: Make file/dir path list - set_fact: - to_delete_paths: "{{ to_delete_paths | default([]) + [item.path] }}" - loop: "{{ files_to_delete.files }}" - when: files_to_delete.files is defined - roles: - - role: cleanup-directories - vars: - directories_files_list_to_remove: "{{ to_delete_paths }}" - when: to_delete_paths is defined diff --git a/ansible/test/play-resources/molecule/nfs/molecule.yml b/ansible/test/play-resources/molecule/nfs/molecule.yml index ffaabb07..9cff6b8d 100644 --- a/ansible/test/play-resources/molecule/nfs/molecule.yml +++ b/ansible/test/play-resources/molecule/nfs/molecule.yml @@ -13,7 +13,7 @@ platforms: override_command: false volumes: - /sys/fs/cgroup:/sys/fs/cgroup:ro - - ${HOME}/data:/data:rw # mount fs from host to get nfs exportfs task working + - /data groups: - resources networks: diff --git a/ansible/test/roles/cleanup-containers/tasks/main.yml b/ansible/test/roles/cleanup-containers/tasks/main.yml deleted file mode 100644 index 3a800c9e..00000000 --- a/ansible/test/roles/cleanup-containers/tasks/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Remove containers - docker_container: - name: "{{ item }}" - state: absent - loop: "{{ container_list }}" diff --git a/ansible/test/roles/cleanup-directories/tasks/main.yml b/ansible/test/roles/cleanup-directories/tasks/main.yml deleted file mode 100644 index 8e79ea0c..00000000 --- a/ansible/test/roles/cleanup-directories/tasks/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- name: Remove directories/files - file: - path: "{{ item }}" - state: absent - become: true - loop: "{{ directories_files_list_to_remove }}" diff --git a/ansible/test/roles/cleanup-rancher/tasks/main.yml b/ansible/test/roles/cleanup-rancher/tasks/main.yml deleted file mode 100644 index 21b02988..00000000 --- a/ansible/test/roles/cleanup-rancher/tasks/main.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# Remove containers spawned by Rancher Agent -- name: Get list of containers spawned by Rancher Agent - docker_list_containers: - label_name: "{{ item.label }}" - label_value: "{{ item.value }}" - loop: "{{ container_list_by_label }}" - register: docker_list_containers_var - -- name: set fact # save a list of containers found by previous task to orphaned_containers var - set_fact: orphaned_containers="{{ orphaned_containers|default([]) + item.containers }}" - loop: "{{ docker_list_containers_var.results }}" - -- name: Remove orphaned containers - docker_container: - name: "{{ item }}" - state: absent - loop: "{{ orphaned_containers }}" diff --git a/ansible/test/roles/prepare-docker/tasks/docker-packages.yml b/ansible/test/roles/prepare-docker/tasks/docker-packages.yml deleted file mode 100644 index 8f55c5ce..00000000 --- a/ansible/test/roles/prepare-docker/tasks/docker-packages.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- - -- name: Download docker related packages - command: yumdownloader --destdir="{{ rpm_dir }}" "{{ docker_ce_rpm }}" - args: - creates: "{{ rpm_dir }}/{{ docker_ce_rpm }}" - -- name: Install docker related packages - yum: - name: "{{ rpm_dir + '/' + docker_ce_rpm + '.rpm'}}" diff --git a/ansible/test/roles/prepare-docker/tasks/docker-socket-override.yml b/ansible/test/roles/prepare-docker/tasks/docker-socket-override.yml deleted file mode 100644 index 0cabadf4..00000000 --- a/ansible/test/roles/prepare-docker/tasks/docker-socket-override.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Create docker.socket systemd dir for override - file: - path: /etc/systemd/system/docker.socket.d - state: directory - -- name: Fake dockerd dependent docker.socket service not to actually listen the docker socket as dockerd is not used in container only docker client - copy: - content: | - [Socket] - ListenStream= - ListenStream=/tmp/fake - dest: /etc/systemd/system/docker.socket.d/override.conf diff --git a/ansible/test/roles/prepare-docker/tasks/enable-repos.yml b/ansible/test/roles/prepare-docker/tasks/enable-repos.yml deleted file mode 100644 index 204bf03d..00000000 --- a/ansible/test/roles/prepare-docker/tasks/enable-repos.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: Enable docker repos back for Molecule testing purposes - copy: - remote_src: yes - src: "{{ item }}" - dest: "{{ (item | splitext)[0] }}" - loop: "{{ docker_needed_repos }}" - -- name: Disable offline repo for molecule testing purposes - lineinfile: - path: "{{ offline_repo_file }}" - regexp: 'enabled = 1' - line: 'enabled = 0' diff --git a/ansible/test/roles/prepare-docker/tasks/main.yml b/ansible/test/roles/prepare-docker/tasks/main.yml deleted file mode 100644 index e7d87061..00000000 --- a/ansible/test/roles/prepare-docker/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: Check repositories - stat: - path: "{{ item }}" - loop: "{{ docker_needed_repos + [offline_repo_file] }}" - register: repos - -- name: Set fact for offline repos created - set_fact: - offline_repo_created: "{{ (repos.results | map(attribute='stat.exists') | list) is all }}" - -- debug: - var: offline_repo_created - -- name: Enable repos back (if package-repository role already setup offline onap repo) for molecule testing purposes to install docker - include_tasks: enable-repos.yml - when: offline_repo_created - -- name: Prepare docker repos normally - include_tasks: prepare-docker-repos.yml - when: not offline_repo_created diff --git a/ansible/test/roles/prepare-docker/tasks/prepare-docker-repos.yml b/ansible/test/roles/prepare-docker/tasks/prepare-docker-repos.yml deleted file mode 100644 index 01ea72c3..00000000 --- a/ansible/test/roles/prepare-docker/tasks/prepare-docker-repos.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- name: Install Docker-CE repo - yum_repository: - name: docker-ce - description: Docker-ce YUM repo - baseurl: https://download.docker.com/linux/centos/7/x86_64/stable/ - gpgcheck: true - gpgkey: https://download.docker.com/linux/centos/gpg - -- name: Create rpm dir - file: - path: "{{ rpm_dir }}" - state: directory - -- name: Handle docker-ce packages - import_tasks: docker-packages.yml - -- name: Fake dockerd on container - import_tasks: docker-socket-override.yml - when: ansible_env.container == 'docker' diff --git a/ansible/test/roles/prepare-docker/vars/main.yml b/ansible/test/roles/prepare-docker/vars/main.yml deleted file mode 100644 index bcd7f365..00000000 --- a/ansible/test/roles/prepare-docker/vars/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -docker_needed_repos: - - /etc/yum.repos.d/CentOS-Base.repo.disabled - - /etc/yum.repos.d/docker-ce.repo.disabled -offline_repo_file: /etc/yum.repos.d/onap.repo -rpm_dir: /root/rpm -docker_ce_rpm: docker-ce-18.09.5-3.el7.x86_64 diff --git a/ansible/test/roles/prepare-helm/defaults/main.yml b/ansible/test/roles/prepare-helm/defaults/main.yml new file mode 100644 index 00000000..8ab9ed3a --- /dev/null +++ b/ansible/test/roles/prepare-helm/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# Helm version to download. +helm_version: 2.12.3 diff --git a/ansible/test/roles/prepare-helm/tasks/main.yml b/ansible/test/roles/prepare-helm/tasks/main.yml new file mode 100644 index 00000000..aa01e281 --- /dev/null +++ b/ansible/test/roles/prepare-helm/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: "Ensure {{ app_data_path }}/downloads directory exists" + file: + path: "{{ app_data_path }}/downloads" + recurse: true + state: directory + +- name: "Download and unarchive helm-{{ helm_version }}" + unarchive: + src: "https://get.helm.sh/helm-v{{ helm_version }}-linux-amd64.tar.gz" + dest: "/tmp" + remote_src: true + +- name: "Copy helm binary" + copy: + src: /tmp/linux-amd64/helm + dest: "{{ app_data_path }}/downloads/helm" + remote_src: true diff --git a/build/download/base.py b/build/download/base.py index 5bcd0ef6..d8b44839 100644 --- a/build/download/base.py +++ b/build/download/base.py @@ -38,7 +38,8 @@ def load_list(item_list): :return: set of items from file """ with open(item_list, 'r') as f: - return {item for item in (line.strip() for line in f) if item} + return {item for item in (line.strip() for line in f) + if item and not item.startswith('#')} def init_progress(items_name): diff --git a/build/download/docker_images.py b/build/download/docker_images.py index e4e742b3..d8138dd6 100755 --- a/build/download/docker_images.py +++ b/build/download/docker_images.py @@ -180,7 +180,7 @@ def download_docker_image(image, save, output_dir, docker_client): if save: save_image(image, pulled_image, output_dir) except Exception as err: - log.error('Error downloading {}: {}'.format(image, err)) + log.exception('Error downloading {}: {}'.format(image, err)) raise err @@ -195,10 +195,10 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): :return: None """ try: - docker_client = docker.client.DockerClient(version='auto') + # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB) + docker_client = docker.client.DockerClient(version='auto', timeout=300) except docker.errors.DockerException as err: - log.error(err) - log.error('Error creating docker client. Check if is docker installed and running' + log.exception('Error creating docker client. Check if is docker installed and running' ' or if you have right permissions.') raise err @@ -221,14 +221,12 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): missing_images['not_saved'] - missing_images['not_pulled'], None, output_dir, docker_client) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} images were not downloaded'.format(error_count)) missing_images = missing(docker_client, target_images, save, output_dir) log.info(check_table(merge_dict_sets(missing_images), missing_images, save)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): @@ -256,11 +254,13 @@ def run_cli(): progress = base.init_progress('Docker images') if not args.check else None try: - sys.exit(download(args.image_list, args.save, args.output_dir, args.check, - progress, args.workers)) + download(args.image_list, args.save, args.output_dir, args.check, + progress, args.workers) except docker.errors.DockerException: - log.error('Irrecoverable error detected.') + log.exception('Irrecoverable error detected.') sys.exit(1) + except RuntimeError as err: + log.exception(err) if __name__ == '__main__': diff --git a/build/download/download.py b/build/download/download.py new file mode 100755 index 00000000..ebce931a --- /dev/null +++ b/build/download/download.py @@ -0,0 +1,158 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import datetime +import timeit + +import base +import docker_images +import git_repos +import http_files +import npm_packages +import rpm_packages + +log = logging.getLogger(name=__name__) + +def parse_args(): + parser=argparse.ArgumentParser(description='Download data from lists') + list_group = parser.add_argument_group() + list_group.add_argument('--docker', action='append', nargs='+', default=[], + metavar=('list', 'dir-name'), + help='Docker type list. If second argument is specified ' + 'it is treated as directory where images will be saved ' + 'otherwise only pull operation is executed') + list_group.add_argument('--http', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='Http type list and directory to save downloaded files') + list_group.add_argument('--npm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='npm type list and directory to save downloaded files') + list_group.add_argument('--rpm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='rpm type list and directory to save downloaded files') + list_group.add_argument('--git', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='git repo type list and directory to save downloaded files') + parser.add_argument('--npm-registry', default='https://registry.npmjs.org', + help='npm registry to use (default: https://registry.npmjs.org)') + parser.add_argument('--check', '-c', action='store_true', default=False, + help='Check what is missing. No download.') + parser.add_argument('--debug', action='store_true', default=False, + help='Turn on debug output') + + args = parser.parse_args() + + for arg in ('docker', 'npm', 'http', 'rpm', 'git'): + if getattr(args, arg): + return args + + parser.error('One of --docker, --npm, --http, --rpm, --git must be specified') + + +def run_cli(): + args = parse_args() + + console_handler = logging.StreamHandler(sys.stdout) + console_formatter = logging.Formatter('%(message)s') + console_handler.setFormatter(console_formatter) + now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + log_file = 'download_data-{}.log'.format(now) + file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s" + + if args.debug: + logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format) + else: + logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format) + root_logger = logging.getLogger() + root_logger.addHandler(console_handler) + + list_with_errors = [] + timer_start = timeit.default_timer() + + for docker_list in args.docker: + log.info('Processing {}.'.format(docker_list[0])) + progress = None if args.check else base.init_progress('docker images') + save = False + if len(docker_list) > 1: + save = True + else: + docker_list.append(None) + try: + docker_images.download(docker_list[0], save, + docker_list[1], args.check, progress) + except RuntimeError: + list_with_errors.append(docker_list[0]) + + for http_list in args.http: + progress = None if args.check else base.init_progress('http files') + log.info('Processing {}.'.format(http_list[0])) + try: + http_files.download(http_list[0], http_list[1], args.check, + progress) + except RuntimeError: + list_with_errors.append(http_list[0]) + + for npm_list in args.npm: + progress = None if args.check else base.init_progress('npm packages') + log.info('Processing {}.'.format(npm_list[0])) + try: + npm_packages.download(npm_list[0], args.npm_registry, npm_list[1], + args.check, progress) + except RuntimeError: + list_with_errors.append(npm_list[0]) + + for rpm_list in args.rpm: + if args.check: + log.info('Check mode for rpm packages is not implemented') + break + log.info('Processing {}.'.format(rpm_list[0])) + try: + rpm_packages.download(rpm_list[0], rpm_list[1]) + except RuntimeError: + list_with_errors.append(rpm_list[0]) + + for git_list in args.git: + if args.check: + log.info('Check mode for git repositories is not implemented') + break + progress = None if args.check else base.init_progress('git repositories') + log.info('Processing {}.'.format(git_list[0])) + try: + git_repos.download(git_list[0], git_list[1], progress) + except RuntimeError: + list_with_errors.append(git_list[0]) + + e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) + log.info(timeit.default_timer() - timer_start) + log.info('Execution ended. Total elapsed time {}'.format(e_time)) + + if list_with_errors: + log.error('Errors encountered while processing these lists:' + '\n{}'.format('\n'.join(list_with_errors))) + sys.exit(1) + + + +if __name__ == '__main__': + run_cli() diff --git a/build/download/git_repos.py b/build/download/git_repos.py index e388e94c..aff01b80 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -45,10 +45,9 @@ def download(git_list, dst_dir, progress): if not base.check_tool('git'): log.error('ERROR: git is not installed') progress.finish(dirty=True) - return 1 + raise RuntimeError('git missing') - git_set = {tuple(item.split()) for item in base.load_list(git_list) - if not item.startswith('#')} + git_set = {tuple(item.split()) for item in base.load_list(git_list)} error_count = 0 @@ -64,14 +63,13 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: - log.error(err.output.decode()) + log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - return error_count - + raise RuntimeError('Download unsuccesfull') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') @@ -85,8 +83,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') progress = base.init_progress('git repositories') - - sys.exit(download(args.git_list, args.output_dir, progress)) + try: + download(args.git_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/http_files.py b/build/download/http_files.py index f5b1e59a..c83158d6 100755 --- a/build/download/http_files.py +++ b/build/download/http_files.py @@ -83,7 +83,7 @@ def download(data_list, dst_dir, check, progress, workers=None): if check: log.info(base.simple_check_table(file_set, missing_files)) - return 0 + return skipping = file_set - missing_files @@ -91,12 +91,11 @@ def download(data_list, dst_dir, check, progress, workers=None): error_count = base.run_concurrent(workers, progress, download_file, missing_files, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} files were not downloaded. Check log for specific failures.'.format(error_count)) + raise RuntimeError() - base.finish_progress(progress, error_count, log) - - return error_count def run_cli(): """ @@ -123,7 +122,10 @@ def run_cli(): progress = base.init_progress('http files') if not args.check else None - sys.exit(download(args.file_list, args.output_dir, args.check, progress, args.workers)) + try: + download(args.file_list, args.output_dir, args.check, progress, args.workers) + except RuntimeError: + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/npm_packages.py b/build/download/npm_packages.py index c174e2c1..70c03ad8 100755 --- a/build/download/npm_packages.py +++ b/build/download/npm_packages.py @@ -57,7 +57,7 @@ def download_npm(npm, registry, dst_dir): except Exception as err: if os.path.isfile(dst_path): os.remove(dst_path) - log.error('Failed: {}: {}'.format(npm, err)) + log.exception('Failed: {}'.format(npm)) raise err log.info('Downloaded: {}'.format(npm)) @@ -81,12 +81,10 @@ def download(npm_list, registry, dst_dir, check_mode, progress=None, workers=Non base.start_progress(progress, len(npm_set), skipping, log) error_count = base.run_concurrent(workers, progress, download_npm, missing_npms, registry, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} packages were not downloaded. Check log for specific failures.'.format(error_count)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): diff --git a/build/download/rpm_packages.py b/build/download/rpm_packages.py index 7f9700a3..732af0ea 100755 --- a/build/download/rpm_packages.py +++ b/build/download/rpm_packages.py @@ -33,7 +33,7 @@ log = logging.getLogger(name=__name__) def download(rpm_list, dst_dir): if not base.check_tool('yumdownloader'): log.error('ERROR: yumdownloader is not installed') - return 1 + raise RuntimeError('yumdownloader missing') rpm_set = base.load_list(rpm_list) @@ -41,11 +41,10 @@ def download(rpm_list, dst_dir): log.info('Running command: {}'.format(command)) try: subprocess.check_call(command.split()) - log.info('Downloaded') except subprocess.CalledProcessError as err: - log.error(err.output) - return err.returncode - + log.exception(err.output) + raise err + log.info('Downloaded') def run_cli(): @@ -59,7 +58,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') - sys.exit(download(args.rpm_list, args.output_dir)) + try: + download(args.rpm_list, args.output_dir) + except (subprocess.CalledProcessError, RuntimeError): + sys.exit(1) + if __name__ == '__main__': diff --git a/build/fetch_and_patch_charts.sh b/build/fetch_and_patch_charts.sh index 79d7a01e..22d45e66 100755 --- a/build/fetch_and_patch_charts.sh +++ b/build/fetch_and_patch_charts.sh @@ -54,7 +54,7 @@ TOTAL=5 PATCH_FILE=$(realpath "${3}") echo -e "${_G}[Step $((CURR++))/${TOTAL} cloning repo with charts to be patched]${C_}" -git clone "${1}" "${4}" +git clone --recurse-submodules "${1}" "${4}" echo -e "${_G}[Step $((CURR++))/${TOTAL} setting working dir to ${4}]${C_}" pushd "${4}" |