diff options
26 files changed, 232 insertions, 88 deletions
diff --git a/ansible/roles/application/.gitignore b/ansible/roles/application/.gitignore deleted file mode 100644 index 155cbb20..00000000 --- a/ansible/roles/application/.gitignore +++ /dev/null @@ -1 +0,0 @@ -application/ diff --git a/ansible/roles/application/molecule/custom_role/Dockerfile.j2 b/ansible/roles/application/molecule/custom_role/Dockerfile.j2 new file mode 120000 index 00000000..867ec5c3 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/Dockerfile.j2 @@ -0,0 +1 @@ +../default/Dockerfile.j2
\ No newline at end of file diff --git a/ansible/roles/application/molecule/custom_role/molecule.yml b/ansible/roles/application/molecule/custom_role/molecule.yml new file mode 100644 index 00000000..f9b29d92 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/molecule.yml @@ -0,0 +1,55 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: instance + image: centos:7 +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + group_vars: + all: + app_name: moleculetestapp + app_data_path: "/opt/{{ app_name }}" + app_helm_release_name: "{{ app_name }}" + app_kubernetes_namespace: "{{ app_name }}" + app_helm_charts_install_directory: application/helm_charts + app_helm_plugins_directory: "{{ app_helm_charts_install_directory}}/helm/plugins/" + app_helm_charts_infra_directory: "{{ app_data_path }}/helm_charts" + helm_bin_dir: /usr/local/bin + app_helm_build_targets: + - all + - onap + app_helm_chart_name: "{{ app_name }}" + application_pre_install_role: application/test-patch-role + application_post_install_role: application/test-patch-role + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml +scenario: + name: custom_role + test_sequence: + - lint + - cleanup + - destroy + - dependency + - syntax + - create + - prepare + - converge + - verify + - cleanup + - destroy +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/application/molecule/custom_role/tests b/ansible/roles/application/molecule/custom_role/tests new file mode 120000 index 00000000..b8ac4407 --- /dev/null +++ b/ansible/roles/application/molecule/custom_role/tests @@ -0,0 +1 @@ +../default/tests/
\ No newline at end of file diff --git a/ansible/roles/application/molecule/default/cleanup.yml b/ansible/roles/application/molecule/default/cleanup.yml new file mode 100644 index 00000000..996acaf1 --- /dev/null +++ b/ansible/roles/application/molecule/default/cleanup.yml @@ -0,0 +1,6 @@ +--- +- name: Cleanup infra + hosts: all + ignore_unreachable: true + roles: + - cleanup-application diff --git a/ansible/roles/application/molecule/default/molecule.yml b/ansible/roles/application/molecule/default/molecule.yml index 8f19d7ff..30c752e2 100644 --- a/ansible/roles/application/molecule/default/molecule.yml +++ b/ansible/roles/application/molecule/default/molecule.yml @@ -27,8 +27,6 @@ provisioner: - all - onap app_helm_chart_name: "{{ app_name }}" - application_pre_install_role: - application_post_install_role: lint: name: ansible-lint scenario: diff --git a/ansible/roles/chrony/molecule/default/playbook.yml b/ansible/roles/chrony/molecule/default/playbook.yml index 7dccfc35..717d0f39 100644 --- a/ansible/roles/chrony/molecule/default/playbook.yml +++ b/ansible/roles/chrony/molecule/default/playbook.yml @@ -1,6 +1,10 @@ --- - name: Converge infrastructure hosts hosts: infrastructure + tasks: + - name: Set cluster_ip fact + set_fact: + cluster_ip: "{{ ansible_default_ipv4.address }}" roles: - chrony diff --git a/ansible/roles/chrony/molecule/default/tests/test_default.py b/ansible/roles/chrony/molecule/default/tests/test_default.py new file mode 100644 index 00000000..08f85d37 --- /dev/null +++ b/ansible/roles/chrony/molecule/default/tests/test_default.py @@ -0,0 +1,42 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +@pytest.fixture +def chrony_conf_file(host, os_family): + conf = host.ansible('include_vars', 'file=../../defaults/main.yml')[ + 'ansible_facts']['chrony']['conf'][os_family]['config_file'] + return conf + + +@pytest.fixture +def os_family(host): + osf = host.ansible("setup")['ansible_facts']['ansible_os_family'] + return osf + + +def test_chrony_conf_file_exists(host, chrony_conf_file): + assert host.file(chrony_conf_file).exists, 'Config file not found!' + + +def test_chrony_service_running_enabled(host): + assert host.service('chronyd').is_running, \ + 'Chronyd service is not running!' + assert host.service('chronyd').is_enabled, \ + 'Chronyd service is not enabled!' + + +def test_ntp_synchronized(host, chrony_conf_file): + assert host.file(chrony_conf_file).exists, 'Config file not found!' + if host.file(chrony_conf_file).contains("server "): + out = host.check_output('systemctl status chronyd') + assert 'Selected source' in out, \ + 'Chronyd did not synchronize with NTP server.' + else: + # Host acts as a time source + pass diff --git a/ansible/roles/chrony/molecule/ubuntu/molecule.yml b/ansible/roles/chrony/molecule/ubuntu/molecule.yml index dd2b9655..e7ede8c7 100644 --- a/ansible/roles/chrony/molecule/ubuntu/molecule.yml +++ b/ansible/roles/chrony/molecule/ubuntu/molecule.yml @@ -37,3 +37,6 @@ verifier: name: testinfra lint: name: flake8 + options: + v: 1 + directory: ../default/tests/ diff --git a/ansible/roles/nexus/.yamllint b/ansible/roles/nexus/.yamllint new file mode 100644 index 00000000..ad0be760 --- /dev/null +++ b/ansible/roles/nexus/.yamllint @@ -0,0 +1,11 @@ +extends: default + +rules: + braces: + max-spaces-inside: 1 + level: error + brackets: + max-spaces-inside: 1 + level: error + line-length: disable + truthy: disable diff --git a/ansible/roles/nexus/defaults/main.yml b/ansible/roles/nexus/defaults/main.yml index 8f636979..194f88bf 100644 --- a/ansible/roles/nexus/defaults/main.yml +++ b/ansible/roles/nexus/defaults/main.yml @@ -6,4 +6,6 @@ populate_nexus: false # but all images are pre-populated either at buildtime or at install time (populate_nexus). runtime_images: {} # Default rule for tarball naming translation +nexus_url: "{{ nexus_url_scheme | default('https') }}://nexus.{{ hostvars[groups.infrastructure[0]].ansible_nodename }}{{ nexus_port | default('') }}" nexus3_image_tar: "{{ nexus3_image | regex_replace('(\\/|\\:)', '_') }}.tar" +nexus3_published_ports: [] diff --git a/ansible/roles/nexus/molecule/default/molecule.yml b/ansible/roles/nexus/molecule/default/molecule.yml new file mode 100644 index 00000000..63c47724 --- /dev/null +++ b/ansible/roles/nexus/molecule/default/molecule.yml @@ -0,0 +1,30 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6} + pre_build_image: true + privileged: true + override_command: false + volumes: + - /var/lib/docker + groups: + - infrastructure +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + links: + group_vars: ../../../../group_vars + lint: + name: ansible-lint +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/nexus/molecule/default/playbook.yml b/ansible/roles/nexus/molecule/default/playbook.yml new file mode 100644 index 00000000..e10ea55b --- /dev/null +++ b/ansible/roles/nexus/molecule/default/playbook.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + vars: + nexus_port: ":8081" + nexus3_published_ports: + - "{{ '8081' + nexus_port }}" + nexus_url_scheme: http + populate_nexus: true + roles: + - nexus diff --git a/ansible/roles/nexus/molecule/default/prepare.yml b/ansible/roles/nexus/molecule/default/prepare.yml new file mode 100644 index 00000000..f7290bd5 --- /dev/null +++ b/ansible/roles/nexus/molecule/default/prepare.yml @@ -0,0 +1,8 @@ +--- +- name: Prepare + hosts: all + vars_files: + - ../../defaults/main.yml + roles: + - prepare-docker-dind + - prepare-nexus diff --git a/ansible/roles/nexus/tasks/install.yml b/ansible/roles/nexus/tasks/install.yml index c88e5855..6ac2b131 100644 --- a/ansible/roles/nexus/tasks/install.yml +++ b/ansible/roles/nexus/tasks/install.yml @@ -24,6 +24,7 @@ image: "{{ nexus3_image }}" networks: - name: nexus_network + published_ports: "{{ nexus3_published_ports }}" volumes: - "{{ app_data_path }}/nexus_data:/nexus-data:rw" state: started diff --git a/ansible/roles/nexus/vars/main.yml b/ansible/roles/nexus/vars/main.yml deleted file mode 100644 index 5ec51869..00000000 --- a/ansible/roles/nexus/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -nexus_url: "https://nexus.{{ hostvars[groups.infrastructure[0]].ansible_nodename }}" diff --git a/ansible/test/play-infrastructure/molecule/default/vars.yml b/ansible/test/play-infrastructure/molecule/default/vars.yml index e91eadb2..6ca5a1ad 100644 --- a/ansible/test/play-infrastructure/molecule/default/vars.yml +++ b/ansible/test/play-infrastructure/molecule/default/vars.yml @@ -16,9 +16,9 @@ molecule_test_registry: nexus3.onap.org:10001 runtime_images: busybox: registry: "{{ molecule_test_registry }}" - path: "/onap/components/busybox" + path: "/busybox" tag: "latest" aaionap-haproxy: registry: "{{ molecule_test_registry }}" - path: "/onap/components/aaionap/haproxy" + path: "/aaionap/haproxy" tag: "1.2.4" diff --git a/ansible/test/roles/cleanup-application/tasks/main.yml b/ansible/test/roles/cleanup-application/tasks/main.yml new file mode 100644 index 00000000..cbb8d521 --- /dev/null +++ b/ansible/test/roles/cleanup-application/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Clean application role mocked artifacts directories + file: + path: "{{ item }}" + state: absent + delegate_to: localhost + loop: + - certs + - application diff --git a/ansible/test/roles/prepare-application/defaults/main.yml b/ansible/test/roles/prepare-application/defaults/main.yml index 227bd4f0..c3883c65 100644 --- a/ansible/test/roles/prepare-application/defaults/main.yml +++ b/ansible/test/roles/prepare-application/defaults/main.yml @@ -1,4 +1,6 @@ --- simulate_helm: true app_helm_charts_install_directory: application/helm_charts -helm_simulation_output_file: /tmp/helm_simu_output
\ No newline at end of file +helm_simulation_output_file: /tmp/helm_simu_output +application_pre_install_role: application/test-patch-role +application_post_install_role: application/test-patch-role diff --git a/ansible/test/roles/prepare-application/tasks/main.yml b/ansible/test/roles/prepare-application/tasks/main.yml index 2f143a07..75abb802 100644 --- a/ansible/test/roles/prepare-application/tasks/main.yml +++ b/ansible/test/roles/prepare-application/tasks/main.yml @@ -1,9 +1,14 @@ --- -- name: Create Application helm charts directory +- name: Create application role mocked artifacts directories file: - path: "{{ app_helm_charts_install_directory }}" + path: "{{ item }}" state: directory delegate_to: localhost + loop: + - "{{ app_helm_charts_install_directory }}" + - certs + - "{{ application_pre_install_role + '/tasks/' }}" + - "{{ application_post_install_role + '/tasks/' }}" - name: Create Makefile to simulate helm charts dir and make building copy: @@ -33,15 +38,19 @@ name: make state: present -- name: Create local certs dir for dummy certs - file: - path: certs - state: directory - delegate_to: localhost - - name: Create dummy cert file to simulate offline server certificates in helm install with override.yml file copy: content: | this is dummy server certificate value dest: certs/rootCA.crt delegate_to: localhost + +- name: Create test patch role files + copy: + content: | + --- + - name: Mocked up patch role + debug: + msg: "Noop task to mock up patch role" + dest: application/test-patch-role/tasks/main.yml + delegate_to: localhost diff --git a/ansible/test/roles/prepare-nexus/tasks/main.yml b/ansible/test/roles/prepare-nexus/tasks/main.yml index 4ef40f5e..5eb1b7d3 100644 --- a/ansible/test/roles/prepare-nexus/tasks/main.yml +++ b/ansible/test/roles/prepare-nexus/tasks/main.yml @@ -29,12 +29,12 @@ docker_image: name: busybox tag: latest - repository: nexus3.onap.org:10001/onap/components/busybox + repository: nexus3.onap.org:10001/busybox - name: Save busybox image delegate_to: localhost docker_image: - name: nexus3.onap.org:10001/onap/components/busybox + name: nexus3.onap.org:10001/busybox tag: latest pull: false archive_path: /tmp/busybox.tar @@ -44,12 +44,12 @@ docker_image: name: aaionap/haproxy tag: 1.2.4 - repository: nexus3.onap.org:10001/onap/components/aaionap/haproxy + repository: nexus3.onap.org:10001/aaionap/haproxy - name: Save haproxy image delegate_to: localhost docker_image: - name: nexus3.onap.org:10001/onap/components/aaionap/haproxy + name: nexus3.onap.org:10001/aaionap/haproxy tag: 1.2.4 pull: false archive_path: /tmp/haproxy.tar diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh index 9206b0bb..6761c328 100755 --- a/build/creating_data/docker-images-collector.sh +++ b/build/creating_data/docker-images-collector.sh @@ -47,7 +47,7 @@ import yaml import sys with open("${1}", 'r') as f: - values = yaml.load(f) + values = yaml.load(f, Loader=yaml.SafeLoader) enabled = filter(lambda x: values[x].get('enabled', False) == True, values) print(' '.join(enabled)) diff --git a/build/data_lists/infra_bin_utils.list b/build/data_lists/infra_bin_utils.list new file mode 100644 index 00000000..8de616e9 --- /dev/null +++ b/build/data_lists/infra_bin_utils.list @@ -0,0 +1,3 @@ +https://storage.googleapis.com/kubernetes-release/release/v1.13.5/bin/linux/amd64/kubectl +https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz +https://github.com/rancher/rke/releases/download/v0.2.1/rke_linux-amd64 diff --git a/build/download/download-bin-tools.sh b/build/download/download-bin-tools.sh deleted file mode 100755 index 327e210f..00000000 --- a/build/download/download-bin-tools.sh +++ /dev/null @@ -1,60 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -usage () { - echo "Usage:" - echo -e "./$(basename $0) [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) ./downloads" -} - -if [ "${1}" == "-h" ] || [ -z "${1}" ] ; then - usage - exit 1 -else - OUTDIR="${1}" -fi - -# we are keeping just dublin support in dublin branch -KUBECTL_VERSION=${KUBECTL_VERSION:-1.13.5} -HELM_VERSION=${HELM_VERSION:-2.12.3} -RKE_VERSION=${RKE_VERSION:-0.2.1} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" - -download() { - url="$1" - url_file="${url%%\?*}" - file=$(basename "$url_file") - echo "Downloading $url" - curl -s --retry 5 -y 10 -Y 10 --location "$url" -o "$file" -} - -download "https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - -download "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" -tar -xf ./helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64/helm -O > helm -rm -f ./helm-v${HELM_VERSION}-linux-amd64.tar.gz - -download "https://github.com/rancher/rke/releases/download/v${RKE_VERSION}/rke_linux-amd64" -mv rke_linux-amd64 rke - -chmod a+x ./helm ./kubectl ./rke - -exit 0 diff --git a/build/download/git_repos.py b/build/download/git_repos.py index aff01b80..1d8c2979 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -21,6 +21,7 @@ import argparse import subprocess +import shutil import logging import sys import os @@ -63,18 +64,20 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: + if os.path.isdir(dst): + shutil.rmtree(dst) log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - raise RuntimeError('Download unsuccesfull') + raise RuntimeError('Download unsuccessful') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') parser.add_argument('git_list', metavar='git-list', - help='File with list of npm packages to download.') + help='File with list of git repos to download.') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst index dfeabb0a..cfddcc99 100644 --- a/docs/BuildGuide.rst +++ b/docs/BuildGuide.rst @@ -102,7 +102,8 @@ so one might try following command to download most of the required artifacts in --git ./build/data_lists/onap_git_repos.list ../resources/git-repo \ --npm ./build/data_lists/onap_npm.list ../resources/offline_data/npm_tar \ --rpm ./build/data_lists/onap_rpm.list ../resources/pkg/rhel \ - --pypi ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi + --pypi ./build/data_lists/onap_pip_packages.list ../resources/offline_data/pypi \ + --http ./build/data_lists/infra_bin_utils.list ../resources/downloads Alternatively, step-by-step procedure is described in Appendix 1. @@ -126,9 +127,13 @@ ToDo: complete and verified list of http files will come just during/after vFWCL :: - # Following step will download and prepare rke, kubectl and helm binaries - # there is some post-processing needed therefore its not very convenient to add support for this step into main download.py script - ./build/download/download-bin-tools.sh ../resources/downloads + # Binaries are downloaded in step one but some post processing is still needed. + # This will be improved in future in installer itself + + tar -xf ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz linux-amd64/helm -O > ../resources/downloads/helm + rm -f ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz + mv ../resources/downloads/rke_linux-amd64 rke + **Step 5 - Create repo** @@ -300,7 +305,10 @@ ToDo: complete and verified list of http files will come just during/after vFWCL :: # Following step will download and prepare rke, kubectl and helm binaries - ./build/download/download-bin-tools.sh ../resources/downloads + ./build/download/download.py --http ./build/data_lists/infra_bin_utils.sh ../resources/downloads + tar -xf ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz linux-amd64/helm -O > ../resources/downloads/helm + rm -f ../resources/downloads/helm-v2.12.3-linux-amd64.tar.gz + mv ../resources/downloads/rke_linux-amd64 rke **Step 7 - rpms** |