diff options
33 files changed, 371 insertions, 168 deletions
diff --git a/ansible/roles/application/.gitignore b/ansible/roles/application/.gitignore deleted file mode 100644 index 155cbb20..00000000 --- a/ansible/roles/application/.gitignore +++ /dev/null @@ -1 +0,0 @@ -application/ diff --git a/ansible/roles/application/molecule/custom_role/Dockerfile.j2 b/ansible/roles/application/molecule/custom_role/Dockerfile.j2 new file mode 120000 index 00000000..867ec5c3 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/Dockerfile.j2 @@ -0,0 +1 @@ +../default/Dockerfile.j2
\ No newline at end of file diff --git a/ansible/roles/application/molecule/custom_role/molecule.yml b/ansible/roles/application/molecule/custom_role/molecule.yml new file mode 100644 index 00000000..f9b29d92 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/molecule.yml @@ -0,0 +1,55 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: instance + image: centos:7 +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + group_vars: + all: + app_name: moleculetestapp + app_data_path: "/opt/{{ app_name }}" + app_helm_release_name: "{{ app_name }}" + app_kubernetes_namespace: "{{ app_name }}" + app_helm_charts_install_directory: application/helm_charts + app_helm_plugins_directory: "{{ app_helm_charts_install_directory}}/helm/plugins/" + app_helm_charts_infra_directory: "{{ app_data_path }}/helm_charts" + helm_bin_dir: /usr/local/bin + app_helm_build_targets: + - all + - onap + app_helm_chart_name: "{{ app_name }}" + application_pre_install_role: application/test-patch-role + application_post_install_role: application/test-patch-role + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml +scenario: + name: custom_role + test_sequence: + - lint + - cleanup + - destroy + - dependency + - syntax + - create + - prepare + - converge + - verify + - cleanup + - destroy +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/application/molecule/custom_role/tests b/ansible/roles/application/molecule/custom_role/tests new file mode 120000 index 00000000..b8ac4407 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/tests @@ -0,0 +1 @@ +../default/tests/
\ No newline at end of file diff --git a/ansible/roles/application/molecule/default/cleanup.yml b/ansible/roles/application/molecule/default/cleanup.yml new file mode 100644 index 00000000..996acaf1 --- /dev/null +++ b/ansible/roles/application/molecule/default/cleanup.yml @@ -0,0 +1,6 @@ +--- +- name: Cleanup infra + hosts: all + ignore_unreachable: true + roles: + - cleanup-application diff --git a/ansible/roles/application/molecule/default/molecule.yml b/ansible/roles/application/molecule/default/molecule.yml index 8f19d7ff..30c752e2 100644 --- a/ansible/roles/application/molecule/default/molecule.yml +++ b/ansible/roles/application/molecule/default/molecule.yml @@ -27,8 +27,6 @@ provisioner: - all - onap app_helm_chart_name: "{{ app_name }}" - application_pre_install_role: - application_post_install_role: lint: name: ansible-lint scenario: diff --git a/ansible/roles/chrony/molecule/default/playbook.yml b/ansible/roles/chrony/molecule/default/playbook.yml index 7dccfc35..717d0f39 100644 --- a/ansible/roles/chrony/molecule/default/playbook.yml +++ b/ansible/roles/chrony/molecule/default/playbook.yml @@ -1,6 +1,10 @@ --- - name: Converge infrastructure hosts hosts: infrastructure + tasks: + - name: Set cluster_ip fact + set_fact: + cluster_ip: "{{ ansible_default_ipv4.address }}" roles: - chrony diff --git a/ansible/roles/chrony/molecule/default/tests/test_default.py b/ansible/roles/chrony/molecule/default/tests/test_default.py new file mode 100644 index 00000000..08f85d37 --- /dev/null +++ b/ansible/roles/chrony/molecule/default/tests/test_default.py @@ -0,0 +1,42 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.fixture +def chrony_conf_file(host, os_family): + conf = host.ansible('include_vars', 'file=../../defaults/main.yml')[ + 'ansible_facts']['chrony']['conf'][os_family]['config_file'] + return conf + + +@pytest.fixture +def os_family(host): + osf = host.ansible("setup")['ansible_facts']['ansible_os_family'] + return osf + + +def test_chrony_conf_file_exists(host, chrony_conf_file): + assert host.file(chrony_conf_file).exists, 'Config file not found!' + + +def test_chrony_service_running_enabled(host): + assert host.service('chronyd').is_running, \ + 'Chronyd service is not running!' + assert host.service('chronyd').is_enabled, \ + 'Chronyd service is not enabled!' + + +def test_ntp_synchronized(host, chrony_conf_file): + assert host.file(chrony_conf_file).exists, 'Config file not found!' + if host.file(chrony_conf_file).contains("server "): + out = host.check_output('systemctl status chronyd') + assert 'Selected source' in out, \ + 'Chronyd did not synchronize with NTP server.' + else: + # Host acts as a time source + pass diff --git a/ansible/roles/chrony/molecule/ubuntu/molecule.yml b/ansible/roles/chrony/molecule/ubuntu/molecule.yml index dd2b9655..e7ede8c7 100644 --- a/ansible/roles/chrony/molecule/ubuntu/molecule.yml +++ b/ansible/roles/chrony/molecule/ubuntu/molecule.yml @@ -37,3 +37,6 @@ verifier: name: testinfra lint: name: flake8 + options: + v: 1 + directory: ../default/tests/ diff --git a/ansible/roles/nexus/.yamllint b/ansible/roles/nexus/.yamllint new file mode 100644 index 00000000..ad0be760 --- /dev/null +++ b/ansible/roles/nexus/.yamllint @@ -0,0 +1,11 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable diff --git a/ansible/roles/nexus/defaults/main.yml b/ansible/roles/nexus/defaults/main.yml index 8f636979..194f88bf 100644 --- a/ansible/roles/nexus/defaults/main.yml +++ b/ansible/roles/nexus/defaults/main.yml @@ -6,4 +6,6 @@ populate_nexus: false # but all images are pre-populated either at buildtime or at install time (populate_nexus). runtime_images: {} # Default rule for tarball naming translation +nexus_url: "{{ nexus_url_scheme | default('https') }}://nexus.{{ hostvars[groups.infrastructure[0]].ansible_nodename }}{{ nexus_port | default('') }}" nexus3_image_tar: "{{ nexus3_image | regex_replace('(\\/|\\:)', '_') }}.tar" +nexus3_published_ports: [] diff --git a/ansible/roles/nexus/molecule/default/molecule.yml b/ansible/roles/nexus/molecule/default/molecule.yml new file mode 100644 index 00000000..63c47724 --- /dev/null +++ b/ansible/roles/nexus/molecule/default/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6} + pre_build_image: true + privileged: true + override_command: false + volumes: + - /var/lib/docker + groups: + - infrastructure +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + links: + group_vars: ../../../../group_vars + lint: + name: ansible-lint +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/nexus/molecule/default/playbook.yml b/ansible/roles/nexus/molecule/default/playbook.yml new file mode 100644 index 00000000..e10ea55b --- /dev/null +++ b/ansible/roles/nexus/molecule/default/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + nexus_port: ":8081" + nexus3_published_ports: + - "{{ '8081' + nexus_port }}" + nexus_url_scheme: http + populate_nexus: true + roles: + - nexus diff --git a/ansible/roles/nexus/molecule/default/prepare.yml b/ansible/roles/nexus/molecule/default/prepare.yml new file mode 100644 index 00000000..f7290bd5 --- /dev/null +++ b/ansible/roles/nexus/molecule/default/prepare.yml @@ -0,0 +1,8 @@ +--- +- name: Prepare + hosts: all + vars_files: + - ../../defaults/main.yml + roles: + - prepare-docker-dind + - prepare-nexus diff --git a/ansible/roles/nexus/tasks/install.yml b/ansible/roles/nexus/tasks/install.yml index c88e5855..6ac2b131 100644 --- a/ansible/roles/nexus/tasks/install.yml +++ b/ansible/roles/nexus/tasks/install.yml @@ -24,6 +24,7 @@ image: "{{ nexus3_image }}" networks: - name: nexus_network + published_ports: "{{ nexus3_published_ports }}" volumes: - "{{ app_data_path }}/nexus_data:/nexus-data:rw" state: started diff --git a/ansible/roles/nexus/vars/main.yml b/ansible/roles/nexus/vars/main.yml deleted file mode 100644 index 5ec51869..00000000 --- a/ansible/roles/nexus/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -nexus_url: "https://nexus.{{ hostvars[groups.infrastructure[0]].ansible_nodename }}" diff --git a/ansible/test/play-infrastructure/molecule/default/vars.yml b/ansible/test/play-infrastructure/molecule/default/vars.yml index e91eadb2..6ca5a1ad 100644 --- a/ansible/test/play-infrastructure/molecule/default/vars.yml +++ b/ansible/test/play-infrastructure/molecule/default/vars.yml @@ -16,9 +16,9 @@ molecule_test_registry: nexus3.onap.org:10001 runtime_images: busybox: registry: "{{ molecule_test_registry }}" - path: "/onap/components/busybox" + path: "/busybox" tag: "latest" aaionap-haproxy: registry: "{{ molecule_test_registry }}" - path: "/onap/components/aaionap/haproxy" + path: "/aaionap/haproxy" tag: "1.2.4" diff --git a/ansible/test/roles/cleanup-application/tasks/main.yml b/ansible/test/roles/cleanup-application/tasks/main.yml new file mode 100644 index 00000000..cbb8d521 --- /dev/null +++ b/ansible/test/roles/cleanup-application/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Clean application role mocked artifacts directories + file: + path: "{{ item }}" + state: absent + delegate_to: localhost + loop: + - certs + - application diff --git a/ansible/test/roles/prepare-application/defaults/main.yml b/ansible/test/roles/prepare-application/defaults/main.yml index 227bd4f0..c3883c65 100644 --- a/ansible/test/roles/prepare-application/defaults/main.yml +++ b/ansible/test/roles/prepare-application/defaults/main.yml @@ -1,4 +1,6 @@ --- simulate_helm: true app_helm_charts_install_directory: application/helm_charts -helm_simulation_output_file: /tmp/helm_simu_output
\ No newline at end of file +helm_simulation_output_file: /tmp/helm_simu_output +application_pre_install_role: application/test-patch-role +application_post_install_role: application/test-patch-role diff --git a/ansible/test/roles/prepare-application/tasks/main.yml b/ansible/test/roles/prepare-application/tasks/main.yml index 2f143a07..75abb802 100644 --- a/ansible/test/roles/prepare-application/tasks/main.yml +++ b/ansible/test/roles/prepare-application/tasks/main.yml @@ -1,9 +1,14 @@ --- -- name: Create Application helm charts directory +- name: Create application role mocked artifacts directories file: - path: "{{ app_helm_charts_install_directory }}" + path: "{{ item }}" state: directory delegate_to: localhost + loop: + - "{{ app_helm_charts_install_directory }}" + - certs + - "{{ application_pre_install_role + '/tasks/' }}" + - "{{ application_post_install_role + '/tasks/' }}" - name: Create Makefile to simulate helm charts dir and make building copy: @@ -33,15 +38,19 @@ name: make state: present -- name: Create local certs dir for dummy certs - file: - path: certs - state: directory - delegate_to: localhost - - name: Create dummy cert file to simulate offline server certificates in helm install with override.yml file copy: content: | this is dummy server certificate value dest: certs/rootCA.crt delegate_to: localhost + +- name: Create test patch role files + copy: + content: | + --- + - name: Mocked up patch role + debug: + msg: "Noop task to mock up patch role" + dest: application/test-patch-role/tasks/main.yml + delegate_to: localhost diff --git a/ansible/test/roles/prepare-nexus/tasks/main.yml b/ansible/test/roles/prepare-nexus/tasks/main.yml index 4ef40f5e..5eb1b7d3 100644 --- a/ansible/test/roles/prepare-nexus/tasks/main.yml +++ b/ansible/test/roles/prepare-nexus/tasks/main.yml @@ -29,12 +29,12 @@ docker_image: name: busybox tag: latest - repository: nexus3.onap.org:10001/onap/components/busybox + repository: nexus3.onap.org:10001/busybox - name: Save busybox image delegate_to: localhost docker_image: - name: nexus3.onap.org:10001/onap/components/busybox + name: nexus3.onap.org:10001/busybox tag: latest pull: false archive_path: /tmp/busybox.tar @@ -44,12 +44,12 @@ docker_image: name: aaionap/haproxy tag: 1.2.4 - repository: nexus3.onap.org:10001/onap/components/aaionap/haproxy + repository: nexus3.onap.org:10001/aaionap/haproxy - name: Save haproxy image delegate_to: localhost docker_image: - name: nexus3.onap.org:10001/onap/components/aaionap/haproxy + name: nexus3.onap.org:10001/aaionap/haproxy tag: 1.2.4 pull: false archive_path: /tmp/haproxy.tar diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh index 9206b0bb..6761c328 100755 --- a/build/creating_data/docker-images-collector.sh +++ b/build/creating_data/docker-images-collector.sh @@ -47,7 +47,7 @@ import yaml import sys with open("${1}", 'r') as f: - values = yaml.load(f) + values = yaml.load(f, Loader=yaml.SafeLoader) enabled = filter(lambda x: values[x].get('enabled', False) == True, values) print(' '.join(enabled)) diff --git a/build/data_lists/infra_bin_utils.list b/build/data_lists/infra_bin_utils.list new file mode 100644 index 00000000..8de616e9 --- /dev/null +++ b/build/data_lists/infra_bin_utils.list @@ -0,0 +1,3 @@ +https://storage.googleapis.com/kubernetes-release/release/v1.13.5/bin/linux/amd64/kubectl +https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz +https://github.com/rancher/rke/releases/download/v0.2.1/rke_linux-amd64 diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list index ab4949b2..48029bc2 100644 --- a/build/data_lists/onap_pip_packages.list +++ b/build/data_lists/onap_pip_packages.list @@ -5,7 +5,7 @@ Flask==1.0.3 idna==2.8 itsdangerous==1.1.0 Jinja2==2.10.1 -MarkupSafe==1.1.1 +MarkupSafe==0.23 requests==2.22.0 urllib3==1.25.3 Werkzeug==0.15.4 diff --git a/build/download/download-bin-tools.sh b/build/download/download-bin-tools.sh deleted file mode 100755 index 327e210f..00000000 --- a/build/download/download-bin-tools.sh +++ /dev/null @@ -1,60 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -usage () { - echo "Usage:" - echo -e "./$(basename $0) [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) ./downloads" -} - -if [ "${1}" == "-h" ] || [ -z "${1}" ] ; then - usage - exit 1 -else - OUTDIR="${1}" -fi - -# we are keeping just dublin support in dublin branch -KUBECTL_VERSION=${KUBECTL_VERSION:-1.13.5} -HELM_VERSION=${HELM_VERSION:-2.12.3} -RKE_VERSION=${RKE_VERSION:-0.2.1} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" - -download() { - url="$1" - url_file="${url%%\?*}" - file=$(basename "$url_file") - echo "Downloading $url" - curl -s --retry 5 -y 10 -Y 10 --location "$url" -o "$file" -} - -download "https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - -download "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" -tar -xf ./helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64/helm -O > helm -rm -f ./helm-v${HELM_VERSION}-linux-amd64.tar.gz - -download "https://github.com/rancher/rke/releases/download/v${RKE_VERSION}/rke_linux-amd64" -mv rke_linux-amd64 rke - -chmod a+x ./helm ./kubectl ./rke - -exit 0 diff --git a/build/download/download-pip.sh b/build/download/download-pip.sh deleted file mode 100755 index 7ab4b0c8..00000000 --- a/build/download/download-pip.sh +++ /dev/null @@ -1,48 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="$1" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi -LIST_FILE=$(readlink -f "$LIST_FILE") - - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -cd "$outdir" -for line in $(clean_list "$LIST_FILE"); do - echo "Downloading $cnt / $lines: $line" - pip download $line - cnt=$((cnt+1)) -done diff --git a/build/download/download.py b/build/download/download.py index ebce931a..0d8912ee 100755 --- a/build/download/download.py +++ b/build/download/download.py @@ -30,6 +30,7 @@ import docker_images import git_repos import http_files import npm_packages +import pypi_packages import rpm_packages log = logging.getLogger(name=__name__) @@ -54,6 +55,9 @@ def parse_args(): list_group.add_argument('--git', action='append', nargs=2, default=[], metavar=('list', 'dir-name'), help='git repo type list and directory to save downloaded files') + list_group.add_argument('--pypi', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='pypi packages type list and directory to save downloaded files') parser.add_argument('--npm-registry', default='https://registry.npmjs.org', help='npm registry to use (default: https://registry.npmjs.org)') parser.add_argument('--check', '-c', action='store_true', default=False, @@ -63,7 +67,7 @@ def parse_args(): args = parser.parse_args() - for arg in ('docker', 'npm', 'http', 'rpm', 'git'): + for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'): if getattr(args, arg): return args @@ -143,6 +147,17 @@ def run_cli(): except RuntimeError: list_with_errors.append(git_list[0]) + for pypi_list in args.pypi: + if args.check: + log.info('Check mode for pypi packages is not implemented') + break + progress = None if args.check else base.init_progress('pypi packages') + log.info('Processing {}.'.format(pypi_list[0])) + try: + pypi_packages.download(pypi_list[0], pypi_list[1], progress) + except RuntimeError: + list_with_errors.append(pypi_list[0]) + e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) log.info(timeit.default_timer() - timer_start) log.info('Execution ended. Total elapsed time {}'.format(e_time)) diff --git a/build/download/git_repos.py b/build/download/git_repos.py index aff01b80..1d8c2979 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -21,6 +21,7 @@ import argparse import subprocess +import shutil import logging import sys import os @@ -63,18 +64,20 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: + if os.path.isdir(dst): + shutil.rmtree(dst) log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - raise RuntimeError('Download unsuccesfull') + raise RuntimeError('Download unsuccessful') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') parser.add_argument('git_list', metavar='git-list', - help='File with list of npm packages to download.') + help='File with list of git repos to download.') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') diff --git a/build/download/pypi_packages.py b/build/download/pypi_packages.py new file mode 100755 index 00000000..951003c5 --- /dev/null +++ b/build/download/pypi_packages.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import subprocess +import os +from retrying import retry + +import base + +log = logging.getLogger(name=__name__) + +@retry(stop_max_attempt_number=5, wait_fixed=5000) +def download_package(package_name, dst_dir): + command = 'pip download --dest {} {}'.format(dst_dir, package_name) + log.info('Running: {}'.format(command)) + log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode()) + log.info('Downloaded: {}'.format(package_name)) + + +def download(pypi_list, dst_dir, progress): + if not base.check_tool('pip'): + log.error('ERROR: pip is not installed') + progress.finish(dirty=True) + raise RuntimeError('pip missing') + + pypi_set = base.load_list(pypi_list) + + error_count = 0 + + base.start_progress(progress, len(pypi_set), [], log) + + for package in pypi_set: + try: + download_package(package, dst_dir) + except subprocess.CalledProcessError as err: + log.exception(err.output.decode()) + error_count += 1 + + progress.update(progress.value + 1) + + base.finish_progress(progress, error_count, log) + if error_count > 0: + log.error('{} packages were not downloaded. Check logs for details'.format(error_count)) + raise RuntimeError('Download unsuccesfull') + + +def run_cli(): + parser = argparse.ArgumentParser(description='Download git repositories from list') + parser.add_argument('pypi_list', metavar='pypi-list', + help='File with list of pypi packages to download.') + parser.add_argument('--output-dir', '-o', default=os.getcwd(), + help='Download destination') + + args = parser.parse_args() + + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') + + progress = base.init_progress('pypi packages') + try: + download(args.pypi_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) + + +if __name__ == '__main__': + run_cli() diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst index 043e4298..cfddcc99 100644 --- a/docs/BuildGuide.rst +++ b/docs/BuildGuide.rst @@ -96,7 +96,14 @@ so one might try following command to download most of the required artifacts in # all resources will be stored in expected folder structure within ../resources folder # for more details refer to Appendix 1. - ./build/download/download.py --docker ./build/data_lists/infra_docker_images.list ../resources/offline_data/docker_images_infra --docker ./build/data_lists/rke_docker_images.list ../resources/offline_data/docker_images_for_nexus --docker ./build/data_lists/onap_docker_images.list ../resources/offline_data/docker_images_for_nexus --git ./build/data_lists/onap_git_repos.list ../resources/git-repo --npm ./build/data_lists/onap_npm.list ../resources/offline_data/npm_tar --rpm ./build/data_lists/onap_rpm.list ../resources/pkg/rhel + ./build/download/download.py --docker ./build/data_lists/infra_docker_images.list ../resources/offline_data/docker_images_infra \ + --docker ./build/data_lists/rke_docker_images.list ../resources/offline_data/docker_images_for_nexus \ + --docker ./build/data_lists/onap_docker_images.list ../resources/offline_data/docker_images_for_nexus \ + --git ./build/data_lists/onap_git_repos.list ../resources/git-repo \ + --npm ./build/data_lists/onap_npm.list ../resources/offline_data/npm_tar \ + --rpm ./build/data_lists/onap_rpm.list ../resources/pkg/rhel \ + --pypi ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi \ + --http ./build/data_lists/infra_bin_utils.list ../resources/downloads Alternatively, step-by-step procedure is described in Appendix 1. @@ -120,24 +127,19 @@ ToDo: complete and verified list of http files will come just during/after vFWCL :: - # Following step will download and prepare rke, kubectl and helm binaries - # there is some post-processing needed therefore its not very convenient to add support for this step into main download.py script - ./build/download/download-bin-tools.sh ../resources/downloads - -**Step 5 - Create repo** - -:: + # Binaries are downloaded in step one but some post processing is still needed. + # This will be improved in future in installer itself - createrepo ../resources/pkg/rhel + tar -xf ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz linux-amd64/helm -O > ../resources/downloads/helm + rm -f ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz + mv ../resources/downloads/rke_linux-amd64 rke -**Step 6 - pip packages** -Todo: will be incorporated into download.py in near future +**Step 5 - Create repo** :: - # Following step will download all pip packages - ./build/download/download-pip.sh ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi + createrepo ../resources/pkg/rhel This concludes SW download part required for ONAP offline platform creating. @@ -268,7 +270,9 @@ Appendix 1. Step-by-step download procedure # This step will parse all 3 docker datalists (offline infrastructure images, rke k8s images & onap images) # and start building onap offline platform in /tmp/resources folder - ./build/download/download.py --docker ./build/data_lists/infra_docker_images.list ../resources/offline_data/docker_images_infra --docker ./build/data_lists/rke_docker_images.list ../resources/offline_data/docker_images_for_nexus --docker ./build/data_lists/onap_docker_images.list ../resources/offline_data/docker_images_for_nexus + ./build/download/download.py --docker ./build/data_lists/infra_docker_images.list ../resources/offline_data/docker_images_infra \ + --docker ./build/data_lists/rke_docker_images.list ../resources/offline_data/docker_images_for_nexus \ + --docker ./build/data_lists/onap_docker_images.list ../resources/offline_data/docker_images_for_nexus **Step 2 - building own dns image** @@ -301,7 +305,10 @@ ToDo: complete and verified list of http files will come just during/after vFWCL :: # Following step will download and prepare rke, kubectl and helm binaries - ./build/download/download-bin-tools.sh ../resources/downloads + ./build/download/download.py --http ./build/data_lists/infra_bin_utils.sh ../resources/downloads + tar -xf ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz linux-amd64/helm -O > ../resources/downloads/helm + rm -f ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz + mv ../resources/downloads/rke_linux-amd64 rke **Step 7 - rpms** @@ -314,10 +321,8 @@ ToDo: complete and verified list of http files will come just during/after vFWCL **Step 8 - pip packages** -Todo: new python script might be created for that part as well - :: # Following step will download all pip packages - ./build/download/download-pip.sh ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi + ./build/download/download.py --pypi ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi diff --git a/helm_deployment_status.py b/helm_deployment_status.py index 8f9a931d..8917e992 100755 --- a/helm_deployment_status.py +++ b/helm_deployment_status.py @@ -25,7 +25,7 @@ import sys import argparse import yaml import requests -from subprocess import Popen,STDOUT,PIPE +from subprocess import Popen,STDOUT,PIPE,check_output import datetime from time import sleep from os.path import expanduser @@ -137,7 +137,7 @@ def check_in_loop(k8s, max_time, sleep_time, verbosity): return ready def check_helm_releases(): - helm = subprocess.check_output(['helm', 'ls']) + helm = check_output(['helm', 'ls']) if helm == '': sys.exit('No Helm releases detected.') helm_releases = csv.DictReader( @@ -180,9 +180,7 @@ def parse_args(): default=expanduser('~') + '/.kube/config', help='path to .kube/config file') parser.add_argument('--health-path', '-hp', help='path to ONAP robot ete-k8s.sh') - parser.add_argument('--health-mode', default='health', help='healthcheck mode', - choices=('health','healthdist','distribute','instantiate','instantiateVFWCL', - 'instantiateDemoVFWCL','portal')) + parser.add_argument('--health-mode', '-hm', default='health', help='healthcheck mode') parser.add_argument('--no-helm', action='store_true', help='Do not check Helm') parser.add_argument('--check-frequency', '-w', default=300, type=int, help='time between readiness checks in seconds') diff --git a/patches/onap-patch-role/tasks/main.yml b/patches/onap-patch-role/tasks/main.yml index 38d0ab92..474b8ed4 100644 --- a/patches/onap-patch-role/tasks/main.yml +++ b/patches/onap-patch-role/tasks/main.yml @@ -12,7 +12,7 @@ stat: path: "{{ app_helm_charts_infra_directory }}/{{ item }}" loop: - - oof/charts/oof-cmso/charts/oof-cmso-service/resources/config/msosimulator.sh + - oof/charts/oof-cmso/charts/oof-cmso-service/templates/deployment.yaml register: hosts_files_check - name: Patch OOM - set npm registry @@ -28,8 +28,8 @@ - name: Patch OOM - nexus domain resolving lineinfile: path: "{{ item.stat.path }}" - regexp: '^(.*)HOSTS_FILE_RECORD' - line: '\g<1>{{ cluster_ip }} {{ simulated_hosts.nexus | join(" ") }} >> /etc/hosts;' + regexp: '^(.*)INFRA_CLUSTER_IP' + line: '\g<1>{{ cluster_ip }}' backrefs: yes state: present loop: "{{ hosts_files_check.results }}" diff --git a/patches/onap.patch b/patches/onap.patch index 05e217e7..a135ad38 100644 --- a/patches/onap.patch +++ b/patches/onap.patch @@ -1,20 +1,29 @@ diff --git a/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/resources/config/msosimulator.sh b/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/resources/config/msosimulator.sh -index c87a26b..79ad921 100644 +index c87a26b..4212c96 100644 --- a/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/resources/config/msosimulator.sh +++ b/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/resources/config/msosimulator.sh -@@ -1,4 +1,10 @@ +@@ -1,4 +1,4 @@ #!/bin/sh -pip install flask -pip install requests -+ -+# for some reason DNS is not working properly on this pod -+# therefore we need to explicitly add record just for -+# the purpose of this script -+HOSTS_FILE_RECORD >> /etc/hosts -+ +pip install -i https://nexus3.onap.org/repository/pypi-private/simple/ --trusted-host nexus3.onap.org flask +pip install -i https://nexus3.onap.org/repository/pypi-private/simple/ --trusted-host nexus3.onap.org requests python /share/etc/config/mock.py +diff --git a/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/templates/deployment.yaml b/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/templates/deployment.yaml +index b26f6dd..2d31aad 100644 +--- a/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/templates/deployment.yaml ++++ b/kubernetes/oof/charts/oof-cmso/charts/oof-cmso-service/templates/deployment.yaml +@@ -30,6 +30,10 @@ spec: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} + spec: ++ hostAliases: ++ - ip: INFRA_CLUSTER_IP ++ hostnames: ++ - nexus3.onap.org + initContainers: + - command: + - /root/ready.py -- diff --git a/kubernetes/common/dgbuilder/templates/deployment.yaml b/kubernetes/common/dgbuilder/templates/deployment.yaml @@ -36,6 +45,6 @@ index 353c231..cf38409 100644 ports: - containerPort: {{ .Values.service.internalPort }} readinessProbe: --- +-- 1.8.3.1 |