diff options
69 files changed, 1203 insertions, 122 deletions
diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..3797dc8b --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,20 @@ +--- +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Required +version: 2 + +formats: + - htmlzip + +build: + image: latest + +python: + version: 3.7 + install: + - requirements: docs/requirements-docs.txt + +sphinx: + configuration: docs/conf.py @@ -66,9 +66,27 @@ committers: company: 'Samsung' id: 'm.zegan' timezone: 'Europe/Warsaw' + - name: 'Sylvain Desbureaux' + email: "sulvain.desbureaux@orange.com" + id: "sdesbure" + company: "Orange" + timezone: 'Europe/Paris' + - name: 'Krzysztof Opasiak' + email: 'k.opasiak@samsung.com' + company: 'Samsung' + id: 'kopasiak' + timezone: 'Europe/Warsaw' tsc: approval: 'https://lists.onap.org/pipermail/onap-tsc' changes: - type: 'Addition' name: 'Bartlomiej Grzybowski' link: 'https://lists.onap.org/g/onap-tsc/topic/32429690' + - type: 'Addition' + name: 'Sylvain Desbureaux' + # yamllint disable-line rule:line-length + link: 'https://lists.onap.org/g/onap-tsc/message/5177?p=,,,20,0,0,0::relevance,,committer+sylvain,20,2,0,32230994' + - type: 'Addition' + name: 'Krzysztof Opasiak' + # yamllint disable-line rule:line-length + link: 'https://lists.onap.org/g/onap-tsc/topic/committer_promotion_request/70242499?p=,,,20,0,0,0::recentpostdate%2Fsticky,,,20,2,0,70242499' diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 0978b814..b92f885f 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -1,9 +1,9 @@ [defaults] - # Define any custom roles used by applications installed by installer # this parameter is telling ansible what additional folder it should # browse while looking up for roles code # relative path ./application is mapped into ansible container under # /ansible/application where application roles should be found roles_path = /ansible/application +# Set log file log_path = /ansible/log/ansible.log diff --git a/ansible/infrastructure.yml b/ansible/infrastructure.yml index 7fdbd2e1..2322c8bb 100644 --- a/ansible/infrastructure.yml +++ b/ansible/infrastructure.yml @@ -8,20 +8,20 @@ - name: Setup infrastructure servers hosts: infrastructure roles: - - chrony - package-repository-check - certificates - docker - dns - vncserver - nginx + - chrony - nexus - name: Setup base for Kubernetes nodes hosts: kubernetes:!infrastructure roles: - - chrony - package-repository-check + - chrony - docker tasks: - include_role: diff --git a/ansible/roles/application/molecule/ubuntu/molecule.yml b/ansible/roles/application/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..2fde35a2 --- /dev/null +++ b/ansible/roles/application/molecule/ubuntu/molecule.yml @@ -0,0 +1,64 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: instance + image: ubuntu:18.04 + dockerfile: ../default/Dockerfile.j2 +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + group_vars: + all: + app_name: moleculetestapp + app_data_path: "/opt/{{ app_name }}" + app_helm_release_name: "{{ app_name }}" + app_kubernetes_namespace: "{{ app_name }}" + app_helm_charts_install_directory: application/helm_charts + app_helm_plugins_directory: "{{ app_helm_charts_install_directory}}/helm/plugins/" + app_helm_charts_infra_directory: "{{ app_data_path }}/helm_charts" + helm_bin_dir: /usr/local/bin + app_helm_build_targets: + - all + - onap + app_helm_chart_name: "{{ app_name }}" + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml +scenario: + name: ubuntu + test_sequence: + - lint + - cleanup + - destroy + - dependency + - syntax + - create + - prepare + - converge + # - idempotence + # --> Action: 'idempotence' + # ERROR: Idempotence test failed because of the following tasks: + # * [instance] => application : Get helm dir + # * [instance] => application : Helm init and upgrade + # * [instance] => application : Helm Serve + # * [instance] => application : Helm Add Repo + # * [instance] => application : Helm Install application moleculetestapp + - side_effect + - verify + - cleanup + - destroy +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests/ diff --git a/ansible/roles/application/tasks/install.yml b/ansible/roles/application/tasks/install.yml index bee01e17..5cffdd07 100644 --- a/ansible/roles/application/tasks/install.yml +++ b/ansible/roles/application/tasks/install.yml @@ -49,6 +49,8 @@ - name: Build local helm repository make: chdir: "{{ app_helm_charts_infra_directory }}" + params: + SKIP_LINT: "TRUE" target: "{{ item }}" loop: "{{ app_helm_build_targets }}" environment: diff --git a/ansible/roles/certificates/defaults/main.yml b/ansible/roles/certificates/defaults/main.yml index ad3422c9..a8bc1769 100644 --- a/ansible/roles/certificates/defaults/main.yml +++ b/ansible/roles/certificates/defaults/main.yml @@ -2,3 +2,11 @@ # Generate certs to local current dir where ansible in run (= playbook_dir) # After ansible run, dir can be deleted but idempotence is lost and certs are re-generated in next run certificates_local_dir: "{{ playbook_dir }}/certs" +root_ca_path: + RedHat: "/etc/pki/ca-trust/source/anchors/" + Debian: "/usr/local/share/ca-certificates/" +extract_root_cert: + RedHat: + update_command: /usr/bin/update-ca-trust extract + Debian: + update_command: update-ca-certificates diff --git a/ansible/roles/certificates/handlers/main.yml b/ansible/roles/certificates/handlers/main.yml index 579b5228..ed80f53f 100644 --- a/ansible/roles/certificates/handlers/main.yml +++ b/ansible/roles/certificates/handlers/main.yml @@ -1,5 +1,5 @@ --- - name: Extract root certificate - command: /usr/bin/update-ca-trust extract + command: "{{ extract_root_cert[ansible_os_family].update_command }}" changed_when: true # this handler is executed just when there is a new cert notify: Restart Docker diff --git a/ansible/roles/certificates/molecule/default/tests/test_default.py b/ansible/roles/certificates/molecule/default/tests/test_default.py index d4314e56..16931fb7 100644 --- a/ansible/roles/certificates/molecule/default/tests/test_default.py +++ b/ansible/roles/certificates/molecule/default/tests/test_default.py @@ -12,8 +12,10 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( ]) def test_cert_file_installed(host, cert_file): os = host.system_info.distribution - if os == "centos": + if (os == "centos"): f = host.file('/etc/pki/ca-trust/source/anchors/' + cert_file) + if (os == "ubuntu"): + f = host.file('/usr/local/share/ca-certificates/' + cert_file) assert f.exists assert f.user == 'root' diff --git a/ansible/roles/certificates/molecule/default/tests/test_infrastructure.py b/ansible/roles/certificates/molecule/default/tests/test_infrastructure.py index 56b12935..6a0aec03 100644 --- a/ansible/roles/certificates/molecule/default/tests/test_infrastructure.py +++ b/ansible/roles/certificates/molecule/default/tests/test_infrastructure.py @@ -27,7 +27,13 @@ def test_generated_cert_files_copied_to_infra(host, cert_file, group_vars): assert f.user == 'root' assert f.group == 'root' + os = host.system_info.distribution + if (os == "centos"): + node_directory = "certs/" + elif (os == "ubuntu"): + node_directory = "../default/certs/" + # Verify cert files content locally is as in node - with open("certs/" + cert_file) as local_cert_file: + with open(node_directory + cert_file) as local_cert_file: local_content = local_cert_file.read().strip() assert local_content == f.content_string diff --git a/ansible/roles/certificates/molecule/ubuntu/.gitignore b/ansible/roles/certificates/molecule/ubuntu/.gitignore new file mode 100644 index 00000000..df912870 --- /dev/null +++ b/ansible/roles/certificates/molecule/ubuntu/.gitignore @@ -0,0 +1 @@ +certs/ diff --git a/ansible/roles/certificates/molecule/ubuntu/group_vars b/ansible/roles/certificates/molecule/ubuntu/group_vars new file mode 120000 index 00000000..5ce8257f --- /dev/null +++ b/ansible/roles/certificates/molecule/ubuntu/group_vars @@ -0,0 +1 @@ +../default/group_vars/
\ No newline at end of file diff --git a/ansible/roles/certificates/molecule/ubuntu/host_vars b/ansible/roles/certificates/molecule/ubuntu/host_vars new file mode 120000 index 00000000..a7046132 --- /dev/null +++ b/ansible/roles/certificates/molecule/ubuntu/host_vars @@ -0,0 +1 @@ +../default/host_vars/
\ No newline at end of file diff --git a/ansible/roles/certificates/molecule/ubuntu/molecule.yml b/ansible/roles/certificates/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..051379df --- /dev/null +++ b/ansible/roles/certificates/molecule/ubuntu/molecule.yml @@ -0,0 +1,69 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - infrastructure + - name: kubernetes-node-1 + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - kubernetes +provisioner: + name: ansible + log: true + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + playbooks: + converge: ../default/playbook.yml + inventory: + links: + group_vars: ../../../../group_vars/ +scenario: + name: ubuntu + test_sequence: + - lint + - cleanup + - destroy + - dependency + - syntax + - create + - prepare + - converge + # - idempotence + # --> Action: 'idempotence' + # ERROR: Idempotence test failed because of the following tasks: + # * [infrastructure-server -> localhost] => certificates : Generate an OpenSSL CSR. + # * [infrastructure-server -> localhost] => certificates : Generate root CA certificate + # * [infrastructure-server] => certificates : Upload certificates to infrastructure server + # * [infrastructure-server] => certificates : Copy root certificate + # * [infrastructure-server] => certificates : Extract root certificate + # * [infrastructure-server] => docker : Setup docker dns settings + # * [kubernetes-node-1] => certificates : Copy root certificate + # * [kubernetes-node-1] => certificates : Extract root certificate + # * [kubernetes-node-1] => certificates : Extract root certificate + - side_effect + - verify + - cleanup + - destroy +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 + directory: ../default/tests diff --git a/ansible/roles/certificates/tasks/upload_root_ca.yml b/ansible/roles/certificates/tasks/upload_root_ca.yml index df50b693..d73446b4 100644 --- a/ansible/roles/certificates/tasks/upload_root_ca.yml +++ b/ansible/roles/certificates/tasks/upload_root_ca.yml @@ -2,6 +2,6 @@ - name: Copy root certificate copy: src: "{{ certificates_local_dir }}/rootCA.crt" - dest: /etc/pki/ca-trust/source/anchors/ + dest: "{{ root_ca_path[ansible_os_family] }}" notify: # handler is triggered just when file is changed - Extract root certificate diff --git a/ansible/roles/chrony/tasks/main.yml b/ansible/roles/chrony/tasks/main.yml index 69a11587..ae95c8e7 100644 --- a/ansible/roles/chrony/tasks/main.yml +++ b/ansible/roles/chrony/tasks/main.yml @@ -1,4 +1,10 @@ --- +- name: Install Chrony - Ubuntu + package: + name: "chrony" + state: present + when: ansible_distribution in ["Ubuntu","Debian"] + - name: Check if server mode set_fact: chrony_mode: 'server' diff --git a/ansible/roles/dns/molecule/ubuntu/group_vars b/ansible/roles/dns/molecule/ubuntu/group_vars new file mode 120000 index 00000000..e04e088f --- /dev/null +++ b/ansible/roles/dns/molecule/ubuntu/group_vars @@ -0,0 +1 @@ +../../../../group_vars/
\ No newline at end of file diff --git a/ansible/roles/dns/molecule/ubuntu/molecule.yml b/ansible/roles/dns/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..5428c04a --- /dev/null +++ b/ansible/roles/dns/molecule/ubuntu/molecule.yml @@ -0,0 +1,42 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: True + privileged: true + override_command: False + groups: + - infrastructure + volumes: + - /var/lib/docker +provisioner: + name: ansible + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml + inventory: + host_vars: + infrastructure-server: + cluster_ip: 127.0.0.1 + group_vars: + all: + app_name: onap + app_data_path: "/opt/{{ app_name }}" +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/dns/tasks/main.yml b/ansible/roles/dns/tasks/main.yml index 8a7f8bca..bfdd83b4 100644 --- a/ansible/roles/dns/tasks/main.yml +++ b/ansible/roles/dns/tasks/main.yml @@ -4,6 +4,12 @@ path: "{{ app_data_path }}/cfg" state: directory +- name: Stop systemd-resolved daemon - Ubuntu + systemd: + name: systemd-resolved + state: stopped + when: ansible_distribution in ["Ubuntu","Debian"] + - name: Create simulated hostnames file template: src: simulated_hosts.j2 diff --git a/ansible/roles/docker/defaults/main.yml b/ansible/roles/docker/defaults/main.yml index 1922f64b..33a86e2d 100644 --- a/ansible/roles/docker/defaults/main.yml +++ b/ansible/roles/docker/defaults/main.yml @@ -2,3 +2,11 @@ docker: log_max_size: 100m log_max_file: 3 +packages: + RedHat: + - python-docker-py + - python-jsonpointer + Debian: + - python3-docker + - python3-json-pointer + - iproute2 diff --git a/ansible/roles/docker/molecule/default/molecule.yml b/ansible/roles/docker/molecule/default/molecule.yml index 1e800ee9..efa7f0ae 100644 --- a/ansible/roles/docker/molecule/default/molecule.yml +++ b/ansible/roles/docker/molecule/default/molecule.yml @@ -29,6 +29,8 @@ provisioner: host_vars: infrastructure-server: cluster_ip: 1.2.3.4 +scenario: + name: default verifier: name: testinfra options: diff --git a/ansible/roles/docker/molecule/default/tests/test_default.py b/ansible/roles/docker/molecule/default/tests/test_default.py index a8adeb65..a73572fd 100644 --- a/ansible/roles/docker/molecule/default/tests/test_default.py +++ b/ansible/roles/docker/molecule/default/tests/test_default.py @@ -1,5 +1,6 @@ import os import pytest +import json import testinfra.utils.ansible_runner @@ -23,13 +24,9 @@ def test_docker_daemon_file(host): assert f.exists assert f.user == 'root' assert f.group == 'root' - assert f.content_string == """{ - "log-opts": { - "max-size": "100m", - "max-file": "3" - }, - "dns": [ - "1.2.3.4" - ], - "log-driver": "json-file" -}""" + print(f.content_string) + json_data = json.loads(f.content_string) + assert json_data["log-driver"] == "json-file" + assert json_data["log-opts"]["max-size"] == "100m" + assert json_data["log-opts"]["max-file"] == "3" + assert json_data["dns"][0] == "1.2.3.4" diff --git a/ansible/roles/docker/molecule/ubuntu/molecule.yml b/ansible/roles/docker/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..8325ffb3 --- /dev/null +++ b/ansible/roles/docker/molecule/ubuntu/molecule.yml @@ -0,0 +1,46 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - infrastructure + networks: + - name: docker_install + purge_networks: true + volumes: + - /var/lib/docker +provisioner: + name: ansible + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + inventory: + host_vars: + infrastructure-server: + cluster_ip: 1.2.3.4 + remote_tmp: $HOME/.ansible/tmp +scenario: + name: ubuntu +verifier: + name: testinfra + options: + verbose: true + lint: + name: flake8 + options: + ignore: W291 # trailing whitespace + directory: ../default/tests/ diff --git a/ansible/roles/docker/tasks/main.yml b/ansible/roles/docker/tasks/main.yml index cd731b11..60060826 100644 --- a/ansible/roles/docker/tasks/main.yml +++ b/ansible/roles/docker/tasks/main.yml @@ -6,16 +6,11 @@ notify: - Restart Docker -- name: Install docker python module +- name: Install required packages package: - name: 'python-docker-py' - state: present - - -- name: Install python jsonpointer module - package: - name: 'python-jsonpointer' + name: "{{ item }}" state: present + loop: "{{ packages[ansible_os_family] }}" - name: Ensure /etc/docker exists file: diff --git a/ansible/roles/helm/molecule/default/molecule.yml b/ansible/roles/helm/molecule/default/molecule.yml index 869f87f6..0d46c2d4 100644 --- a/ansible/roles/helm/molecule/default/molecule.yml +++ b/ansible/roles/helm/molecule/default/molecule.yml @@ -26,6 +26,8 @@ provisioner: app_name: onap app_data_path: "/opt/{{ app_name }}" helm_bin_dir: /usr/local/bin +scenario: + name: default verifier: name: testinfra lint: diff --git a/ansible/roles/helm/molecule/ubuntu/group_vars b/ansible/roles/helm/molecule/ubuntu/group_vars new file mode 120000 index 00000000..5ce8257f --- /dev/null +++ b/ansible/roles/helm/molecule/ubuntu/group_vars @@ -0,0 +1 @@ +../default/group_vars/
\ No newline at end of file diff --git a/ansible/roles/helm/molecule/ubuntu/molecule.yml b/ansible/roles/helm/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..a375a32d --- /dev/null +++ b/ansible/roles/helm/molecule/ubuntu/molecule.yml @@ -0,0 +1,38 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: True + privileged: true + override_command: False + groups: + - infrastructure +provisioner: + name: ansible + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + inventory: + group_vars: + all: + app_name: onap + app_data_path: "/opt/{{ app_name }}" + helm_bin_dir: /usr/local/bin +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests diff --git a/ansible/roles/kubectl/molecule/default/molecule.yml b/ansible/roles/kubectl/molecule/default/molecule.yml index bffb29e6..040564e2 100644 --- a/ansible/roles/kubectl/molecule/default/molecule.yml +++ b/ansible/roles/kubectl/molecule/default/molecule.yml @@ -25,6 +25,8 @@ provisioner: all: app_name: onap app_data_path: "/opt/{{ app_name }}" +scenario: + name: default verifier: name: testinfra lint: diff --git a/ansible/roles/kubectl/molecule/ubuntu/molecule.yml b/ansible/roles/kubectl/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..1b2c9f0e --- /dev/null +++ b/ansible/roles/kubectl/molecule/ubuntu/molecule.yml @@ -0,0 +1,37 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: True + privileged: true + override_command: False + groups: + - infrastructure +provisioner: + name: ansible + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + ANSIBLE_LIBRARY: ../../../../library + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + inventory: + group_vars: + all: + app_name: onap + app_data_path: "/opt/{{ app_name }}" +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests/ diff --git a/ansible/roles/nexus/molecule/default/molecule.yml b/ansible/roles/nexus/molecule/default/molecule.yml index 63c47724..e38640d4 100644 --- a/ansible/roles/nexus/molecule/default/molecule.yml +++ b/ansible/roles/nexus/molecule/default/molecule.yml @@ -24,6 +24,8 @@ provisioner: group_vars: ../../../../group_vars lint: name: ansible-lint +scenario: + name: default verifier: name: testinfra lint: diff --git a/ansible/roles/nexus/molecule/ubuntu/molecule.yml b/ansible/roles/nexus/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..9955e7d5 --- /dev/null +++ b/ansible/roles/nexus/molecule/ubuntu/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + override_command: false + volumes: + - /var/lib/docker + groups: + - infrastructure +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + links: + group_vars: ../../../../group_vars + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/nfs/defaults/main.yml b/ansible/roles/nfs/defaults/main.yml index bce98da6..adeaf919 100644 --- a/ansible/roles/nfs/defaults/main.yml +++ b/ansible/roles/nfs/defaults/main.yml @@ -2,7 +2,18 @@ nfs_packages: RedHat: - nfs-utils + Debian: + - nfs-common + - nfs-kernel-server nfs_services: RedHat: - rpcbind - nfs-server + Debian: + - rpcbind + - nfs-kernel-server +nfs_destination: + RedHat: + - "/etc/exports.d/dockerdata-nfs.exports" + Debian: + - "/etc/exports" diff --git a/ansible/roles/nfs/molecule/default/molecule.yml b/ansible/roles/nfs/molecule/default/molecule.yml index a8ca6a30..9af32360 100644 --- a/ansible/roles/nfs/molecule/default/molecule.yml +++ b/ansible/roles/nfs/molecule/default/molecule.yml @@ -43,6 +43,8 @@ provisioner: host_vars: host_vars lint: name: ansible-lint +scenario: + name: default verifier: name: testinfra lint: diff --git a/ansible/roles/nfs/molecule/default/tests/test_default.py b/ansible/roles/nfs/molecule/default/tests/test_default.py index 48139898..dc808753 100644 --- a/ansible/roles/nfs/molecule/default/tests/test_default.py +++ b/ansible/roles/nfs/molecule/default/tests/test_default.py @@ -7,10 +7,13 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') -@pytest.mark.parametrize('pkg', [ - 'nfs-utils' +@pytest.mark.parametrize('distro,pkg', [ + ('centos', 'nfs-utils'), + ('ubuntu', 'nfs-common'), + ('ubuntu', 'nfs-kernel-server') ]) -def test_pkg(host, pkg): - package = host.package(pkg) - - assert package.is_installed +def test_pkg(host, distro, pkg): + os = host.system_info.distribution + if distro == os: + package = host.package(pkg) + assert package.is_installed diff --git a/ansible/roles/nfs/molecule/default/tests/test_nfs-server.py b/ansible/roles/nfs/molecule/default/tests/test_nfs-server.py index 88ba0a61..e35e21c3 100644 --- a/ansible/roles/nfs/molecule/default/tests/test_nfs-server.py +++ b/ansible/roles/nfs/molecule/default/tests/test_nfs-server.py @@ -20,9 +20,14 @@ def test_svc(host, svc): def test_exports(host): + os = host.system_info.distribution + if (os == "centos"): + host_file = "/etc/exports.d/dockerdata-nfs.exports" + elif (os == "ubuntu"): + host_file = "/etc/exports" node2_ip = testinfra.get_host("docker://kubernetes-node-2").interface( "eth0").addresses[0] - f = host.file("/etc/exports.d/dockerdata-nfs.exports") + f = host.file(host_file) assert f.exists assert f.content_string == \ """/dockerdata-nfs """ + node2_ip + """(rw,sync,no_root_squash,no_subtree_check)""" # noqa: E501 diff --git a/ansible/roles/nfs/molecule/ubuntu/molecule.yml b/ansible/roles/nfs/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..3fe393fc --- /dev/null +++ b/ansible/roles/nfs/molecule/ubuntu/molecule.yml @@ -0,0 +1,54 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: kubernetes-node-1 + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - kubernetes + - nfs-server + purge_networks: true + networks: + - name: nfs-net + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /dockerdata-nfs + - name: kubernetes-node-2 + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - kubernetes + purge_networks: true + networks: + - name: nfs-net + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: "../../../../test/roles" + inventory: + links: + group_vars: ../../../../group_vars + host_vars: ../default/host_vars + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests diff --git a/ansible/roles/nfs/tasks/main.yml b/ansible/roles/nfs/tasks/main.yml index 1d848876..cc5290db 100644 --- a/ansible/roles/nfs/tasks/main.yml +++ b/ansible/roles/nfs/tasks/main.yml @@ -23,7 +23,8 @@ - name: Add hosts to exports template: src: exports.j2 - dest: /etc/exports.d/dockerdata-nfs.exports + dest: "{{ item }}" + loop: "{{ nfs_destination[ansible_os_family] }}" notify: - reload nfs when: diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml index c2f1e05c..1269783d 100644 --- a/ansible/roles/nginx/defaults/main.yml +++ b/ansible/roles/nginx/defaults/main.yml @@ -5,7 +5,9 @@ simulated_hosts: nexus: all_simulated_hosts: "{{ simulated_hosts.git + simulated_hosts.http + simulated_hosts.nexus }}" - +package_type: + RedHat: rpm + Debian: deb nginx: ports: - "80:80" @@ -16,7 +18,7 @@ nginx: - "{{ app_data_path }}/certs:/etc/nginx/certs:ro" - "{{ app_data_path }}/git-repo:/srv/git:rw" - "{{ app_data_path }}/http:/srv/http:rw" - - "{{ app_data_path }}/pkg/rpm:/srv/http/repo.infra-server/rpm:rw" + - "{{ app_data_path }}/pkg/{{ package_type[ansible_os_family] }}:/srv/http/repo.infra-server/{{ package_type[ansible_os_family] }}:rw" - "{{ app_data_path }}/pkg/ubuntu/xenial:/srv/http/repo.infra-server/ubuntu/xenial:rw" - /var/log/nginx:/var/log/nginx:rw # Default rule for tarball naming translation diff --git a/ansible/roles/nginx/molecule/ubuntu/molecule.yml b/ansible/roles/nginx/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..9955e7d5 --- /dev/null +++ b/ansible/roles/nginx/molecule/ubuntu/molecule.yml @@ -0,0 +1,36 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + override_command: false + volumes: + - /var/lib/docker + groups: + - infrastructure +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles + inventory: + links: + group_vars: ../../../../group_vars + lint: + name: ansible-lint + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml + cleanup: ../default/cleanup.yml +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 diff --git a/ansible/roles/package-repository/handlers/main.yml b/ansible/roles/package-repository/handlers/main.yml new file mode 100644 index 00000000..304cc873 --- /dev/null +++ b/ansible/roles/package-repository/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart NetworkManager + systemd: + name: NetworkManager + state: restarted + when: ansible_connection != 'docker' diff --git a/ansible/roles/package-repository/tasks/main.yml b/ansible/roles/package-repository/tasks/main.yml index 7dc2e76f..e2a5fd46 100644 --- a/ansible/roles/package-repository/tasks/main.yml +++ b/ansible/roles/package-repository/tasks/main.yml @@ -1,4 +1,20 @@ --- +- name: Disable DNS management in Network Manager + ini_file: + path: /etc/NetworkManager/NetworkManager.conf + state: present + no_extra_spaces: true + section: main + option: dns + value: none + owner: root + group: root + mode: 0644 + backup: false + when: ansible_os_family == 'RedHat' + notify: + - Restart NetworkManager + - name: Setup resolv.conf for node to find package repository by name from infra lineinfile: line: "nameserver {{ hostvars[groups.infrastructure[0]].cluster_ip }}" diff --git a/ansible/roles/resource-data/molecule/ubuntu/group_vars b/ansible/roles/resource-data/molecule/ubuntu/group_vars new file mode 120000 index 00000000..5ce8257f --- /dev/null +++ b/ansible/roles/resource-data/molecule/ubuntu/group_vars @@ -0,0 +1 @@ +../default/group_vars/
\ No newline at end of file diff --git a/ansible/roles/resource-data/molecule/ubuntu/molecule.yml b/ansible/roles/resource-data/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..7f0eb4e4 --- /dev/null +++ b/ansible/roles/resource-data/molecule/ubuntu/molecule.yml @@ -0,0 +1,51 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + + - name: resource-host + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - resources + networks: + - name: resource-data + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + # - ${HOME}/resource-data:/data:rw # mount fs from host to get nfs exportfs task working + + - name: infrastructure-server + image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04} + pre_build_image: true + privileged: true + command: ${MOLECULE_DOCKER_COMMAND:-""} + groups: + - infrastructure + networks: + - name: resource-data + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + +provisioner: + name: ansible + log: true + lint: + name: ansible-lint + env: + ANSIBLE_ROLES_PATH: ../../../../test/roles/ + playbooks: + prepare: ../default/prepare.yml + converge: ../default/playbook.yml +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests diff --git a/ansible/roles/setup/molecule/ubuntu/molecule.yml b/ansible/roles/setup/molecule/ubuntu/molecule.yml new file mode 100644 index 00000000..16dcedf7 --- /dev/null +++ b/ansible/roles/setup/molecule/ubuntu/molecule.yml @@ -0,0 +1,24 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint +platforms: + - name: instance + image: ${PLATFORM_DISTRO:-ubuntu}:${DISTRO_VERSION:-18.04} + dockerfile: ../default/Dockerfile.j2 +provisioner: + name: ansible + lint: + name: ansible-lint + playbooks: + converge: ../default/playbook.yml +scenario: + name: ubuntu +verifier: + name: testinfra + lint: + name: flake8 + directory: ../default/tests/ diff --git a/ansible/test/images/docker/ubuntu/Dockerfile b/ansible/test/images/docker/ubuntu/Dockerfile index 54416374..6dd079ad 100644 --- a/ansible/test/images/docker/ubuntu/Dockerfile +++ b/ansible/test/images/docker/ubuntu/Dockerfile @@ -4,8 +4,11 @@ FROM ubuntu:${RELEASE} # Systemd requires this env for ConditionVirtualization setting in unit files ENV container docker -# Python2.7 required by ansible -RUN apt-get update && apt-get -y install dbus systemd python openssh-server +# Install necessary packages +RUN apt-get update && apt-get -y install dbus systemd openssh-server iproute2 python3-docker + +# Create symlink python3 -> python +RUN ln -s /usr/bin/python3 /usr/bin/python EXPOSE 22 diff --git a/ansible/test/roles/prepare-docker-dind/tasks/main.yml b/ansible/test/roles/prepare-docker-dind/tasks/main.yml index c0bf1543..50efe143 100644 --- a/ansible/test/roles/prepare-docker-dind/tasks/main.yml +++ b/ansible/test/roles/prepare-docker-dind/tasks/main.yml @@ -1,24 +1,6 @@ --- -# Needed because host system has all mounts by default to shared, and -# some things may depend on mounts being shared if we run docker inside -# test env. -- name: "Make all mounts shared" - command: "mount --make-rshared /" - args: - warn: false +- include: rhel.yml + when: ansible_distribution in ["CentOS","Red Hat Enterprise Linux"] -- name: "Enable docker repository" - yum_repository: - name: "Docker" - description: Docker-ce repository - enabled: yes - baseurl: "https://download.docker.com/linux/centos/7/$basearch/stable" - gpgcheck: yes - gpgkey: https://download.docker.com/linux/centos/gpg - -- name: "Install docker" - package: - name: "docker-ce-{{ docker_version }}" - state: present - allow_downgrade: true - notify: Restart docker +- include: ubuntu.yml + when: ansible_distribution in ["Ubuntu","Debian"]
\ No newline at end of file diff --git a/ansible/test/roles/prepare-docker-dind/tasks/rhel.yml b/ansible/test/roles/prepare-docker-dind/tasks/rhel.yml new file mode 100644 index 00000000..4184ef05 --- /dev/null +++ b/ansible/test/roles/prepare-docker-dind/tasks/rhel.yml @@ -0,0 +1,24 @@ +--- +# Needed because host system has all mounts by default to shared, and +# some things may depend on mounts being shared if we run docker inside +# test env. +- name: "Make all mounts shared" + command: "mount --make-rshared /" + args: + warn: false + +- name: "Enable docker repository - yum" + yum_repository: + name: "Docker" + description: Docker-ce repository + enabled: yes + baseurl: "https://download.docker.com/linux/centos/7/$basearch/stable" + gpgcheck: yes + gpgkey: https://download.docker.com/linux/centos/gpg + +- name: "Install docker" + package: + name: "docker-ce-{{ docker_version }}" + state: present + allow_downgrade: true + notify: Restart docker diff --git a/ansible/test/roles/prepare-docker-dind/tasks/ubuntu.yml b/ansible/test/roles/prepare-docker-dind/tasks/ubuntu.yml new file mode 100644 index 00000000..a41c4c20 --- /dev/null +++ b/ansible/test/roles/prepare-docker-dind/tasks/ubuntu.yml @@ -0,0 +1,33 @@ +--- +# Needed because host system has all mounts by default to shared, and +# some things may depend on mounts being shared if we run docker inside +# test env. +- name: "Make all mounts shared" + command: "mount --make-rshared /" + args: + warn: false + +- name: "Install GNUPG for apt-key" + package: + name: "gnupg" + state: present + +- name: "Add an apt key" + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + state: present + +- name: "Enable docker repository - apt" + apt_repository: + repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" + state: present + validate_certs: true + filename: "Docker" + +- name: "Install docker - apt" + apt: + name: "docker-ce" + state: present + update_cache: true + notify: Restart docker diff --git a/build/create_repo.sh b/build/create_repo.sh index eaf0ee30..fa53e688 100755 --- a/build/create_repo.sh +++ b/build/create_repo.sh @@ -1,26 +1,66 @@ #!/usr/bin/env bash -container_name="centos_repo" +# Set type of distribution +distro_type="$(cat /etc/*-release | grep -w "ID" | awk -F'=' '{ print $2 }' | tr -d '"')" + # Path to folder with cloned offline-installer build directory with docker_entrypoint script volume_offline_directory="$(readlink -f $(dirname ${0}))" + # Path for directory where repository will be created volume_repo_directory="$(pwd)" + # Path inside container with cloned offline-installer build directory container_offline_volume="/mnt/offline/" + # Path inside container where will be created repository container_repo_volume="/mnt/repo/" -# Docker image name and version -docker_image="centos:centos7.6.1810" -# Expected directory for RPM packages -expected_dir="resources/pkg/rpm" +# Path inside container where will be stored additional packages lists +container_list_volume="/mnt/additional-lists/" + +# Show help for using this script help () { - echo "Script for run docker container with RPM repository" - echo "usage: create_repo.sh [-d|--destination-repository output directory] [-c|--cloned-directory input directory]" - echo "-h --help: Show this help" - echo "-d --destination-repository: set path where will be stored RPM packages. Default value is current directory" - echo "-c --cloned-directory: set path where is stored this script and docker-entrypoint script (offline-installer/build directory). Fill it just when you want to use different script/datalists" - echo "If build folder from offline repository is not specified will be used default path of current folder." +cat <<EOF +Script for run docker container creating DEB or RPM repository + +Type of repository is created based on user input or if input is empty type of host OS + +usage: create_repo.sh [-d|--destination-repository output directory] [-c|--cloned-directory input directory] + [-t|--target-platform centos target platform for repository] + [-a|----additional-lists path to additional package list] +-h --help: Show this help +-d --destination-repository: set path where will be stored RPM packages. Default value is current directory +-c --cloned-directory: set path where is stored this script and docker-entrypoint script (offline-installer/build directory). Fill it just when you want to use different script/datalists +-t --target-platform: set target platform for repository (ubuntu/rhel/centos) +-a --additional-list: add additional packages list + can be used multiple times for more additional lists + +If build folder from offline repository is not specified will be used default path of current folder. +EOF +} + +# Get type of distribution +# Set Docker image name and version based on type of linux distribution +# Set expected directory for RPM/DEB packages +set_enviroment () { + case "$1" in + ubuntu) + distro_type="ubuntu" + docker_image="ubuntu:18.04" + expected_dir="resources/pkg/deb" + container_name=$1"_repo" + ;; + centos|rhel) + distro_type="rhel" + docker_image="centos:centos7.6.1810" + expected_dir="resources/pkg/rpm" + container_name=$1"_repo" + ;; + *) + echo "Unknown type of linux distribution." + exit 1 + ;; + esac } # Getting input parametters @@ -29,6 +69,7 @@ if [[ $# -eq 0 ]] ; then help # show help exit 0 fi + while [[ $# -gt 0 ]] do case "$1" in @@ -47,6 +88,16 @@ do # Sets path where will be repository created volume_repo_directory="$2" ;; + -t|--target-platform) + # Repository type (rpm/deb) + # Sets target platform for repository + target_input="$2" + ;; + -a|--additional-list) + # Array with more packages lists + # Add more packages lists to download + additional_lists+=("$2") + ;; *) # unknown option help # show help @@ -56,10 +107,28 @@ do shift;shift done -# Check if path contains expected path "resources/pkg/rpm" +# Check if user specified type of repository +# This settings have higher priority, then type of distribution +if ! test -z "$target_input" +then + set_enviroment "$target_input" +else + set_enviroment "$distro_type" +fi + +# Check if path contains expected path: +# "resources/pkg/rpm" for Rhel/CentOS or +# "resources/pkg/deb" for Ubuntu/Debian if ! [[ "/$volume_repo_directory/" = *"/$expected_dir/"* ]]; then # Create repo folder if it not exists - volume_repo_directory="$volume_repo_directory"/resources/pkg/rpm + case "$distro_type" in + ubuntu) + volume_repo_directory="$volume_repo_directory"/resources/pkg/deb + ;; + rhel) + volume_repo_directory="$volume_repo_directory"/resources/pkg/rhel + ;; + esac [ ! -d "$volume_repo_directory" ] && mkdir -p $volume_repo_directory fi @@ -72,15 +141,28 @@ if [ ! "$(docker ps -q -f name=$container_name)" ]; then # run repo container # name of container $container_name # docker entrypoint script from mounted volume - # + # with dynamic parameters + # mount additional packages lists to container + param_array=() + mounted_lists=() + param_array+=(--directory ${container_repo_volume}) + param_array+=(--list ${container_offline_volume}data_lists/) + param_array+=(--packages-lists-path ${container_list_volume}) + [[ ! ${#additional_lists[@]} -eq 0 ]] && \ + for array_list in "${additional_lists[@]}"; + do + param_array+=(--additional-list "${array_list##*/}") && \ + mounted_lists+=(-v ${array_list}:${container_list_volume}${array_list##*/}) + done + docker run -d \ --name $container_name \ -v ${volume_offline_directory}:${container_offline_volume} \ -v ${volume_repo_directory}:${container_repo_volume} \ + "${mounted_lists[@]}" \ --rm \ --entrypoint="${container_offline_volume}docker-entrypoint.sh" \ - -it ${docker_image} \ - --directory ${container_repo_volume} \ - --list ${container_offline_volume}data_lists/ - docker logs $(docker ps --filter "name=centos_repo" --format '{{.ID}}' -a) -f + -it ${docker_image} \ + "${param_array[@]}" + docker logs $(docker ps --filter "name=${container_name}" --format '{{.ID}}' -a) -f fi diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh index c07de107..76ee9016 100755 --- a/build/creating_data/docker-images-collector.sh +++ b/build/creating_data/docker-images-collector.sh @@ -40,15 +40,13 @@ usage () { } parse_yaml() { -python - <<PYP -#!/usr/bin/python -from __future__ import print_function +python3 - <<PYP +#!/usr/bin/python3 import yaml import sys with open("${1}", 'r') as f: values = yaml.load(f, Loader=yaml.SafeLoader) - enabled = filter(lambda x: values[x].get('enabled', False) == True, values) print(' '.join(enabled)) PYP diff --git a/build/data_lists/additional_packages.list b/build/data_lists/additional_packages.list index 98d1dda1..e43bd52e 100644 --- a/build/data_lists/additional_packages.list +++ b/build/data_lists/additional_packages.list @@ -1,2 +1,3 @@ jq screen +git diff --git a/build/data_lists/onap_deb.list b/build/data_lists/onap_deb.list new file mode 100644 index 00000000..fcc6391d --- /dev/null +++ b/build/data_lists/onap_deb.list @@ -0,0 +1,57 @@ +docker-ce=5:18.09.5~3-0~ubuntu-bionic +docker-ce-cli=5:18.09.5~3-0~ubuntu-bionic +containerd.io=1.2.2-3 +pigz +libltdl7 +cgroupfs-mount +aufs-tools +bridge-utils +runc +ubuntu-fan +golang-docker-credential-helpers +libsecret-common +python3-docker +python3-dockerpycreds +python3-websocket +gssproxy +libbasicobjects0 +libcollection4 +libgssrpc4 +libini-config5 +libpath-utils1 +libref-array1 +libverto1 +libverto-libevent1 +keyutils +libnfsidmap2 +libtirpc1 +nfs-common +nfs-kernel-server +rpcbind +chrony +libnspr4 +build-essential +cpp +dpkg-dev +g++ +g++-7 +libcc-0 +libcc-7-dev +libgomp1 +libitm1 +libatomic1 +libasan4 +liblsan0 +libtsan0 +libubsan0 +libcilkrts5 +libmpx2 +libquadmath0 +libc6-dev +gcc +gcc-7 +libc6-dev +libc-dev +make +binutils +resolvconf diff --git a/build/docker-entrypoint.sh b/build/docker-entrypoint.sh index 14f6aaa7..b3306e26 100755 --- a/build/docker-entrypoint.sh +++ b/build/docker-entrypoint.sh @@ -1,21 +1,47 @@ #!/usr/bin/env bash +# Set type of distribution where script is running +distro_type=$(cat /etc/*-release | grep -w "ID" | awk -F'=' '{ print $2 }' | tr -d '"') +case "$distro_type" in + ubuntu) + distro_type="ubuntu" + ;; + rhel|centos) + distro_type="rhel" + ;; + *) + echo "Unknown type of linux distribution." + exit 1 + ;; +esac + # Path where will be created repository (in container) OFFLINE_REPO_DIR="" -# Path where is stored onap_rpm.list file -RPM_LIST_DIR="" +# Path where is stored onap_rpm.list and onap_deb.list file +PCKG_LIST_DIR="" + +# Path where is stored additional packages lists +ADD_LIST_DIR="" +# Show help for using this script help () { - echo -e "Docker entrypoint script for creating RPM repository\n" - echo "usage: create-repo.sh [-d|--directory output directory] [-l|--list input rpm list directory]" - echo "-h --help: Show this help" - echo "-d --directory: set path for repo directory in container" - echo -e "-l --list: set path where rpm list is stored in container\n" - echo "Both paths have to be set with shared volume between" - echo "container and host computer. Default path in container is: /tmp/" - echo "Repository will be created at: /<path>/resources/pkg/rpm/" - echo "RMP list is stored at: ./data_list/" +cat <<EOF +Docker entrypoint script for creating RPM/DEB repository based on linux distribution where script is running + +usage: create-repo.sh [-d|--directory output directory] [-l|--list input rpm/deb list directory] [-a|--additional-lists list1.list] +-h --help: Show this help +-d --directory: set path for repo directory in container +-l --list: set path where rpm or deb list is stored in container +-a --additional-list: add name of additional packages list + can be used multiple times for more additional lists +-p --packages-lists-path: set path for other additional packages lists + +Both paths have to be set with shared volume between +container and host computer. Default path in container is: /tmp/ +Repository will be created at: /<path>/resources/pkg/rhel/ +RMP/DEB list is stored at: ./data_list/ +EOF } # Getting input parametters @@ -39,8 +65,17 @@ do ;; -l|--list) # List parametter - # Sets path where is stored onap_rpm.list file - RPM_LIST_DIR="$2" + # Sets path where is stored onap_rpm.list or onap_deb.list file + PCKG_LIST_DIR="$2" + ;; + -p|--packages-lists-path) + # Path parametter + # Sets path where is stored additional packages lists + ADD_LIST_DIR="$2" + ;; + -a|--additional-list) + # Array of additional packages lists + ADDITIONAL_LISTS+=("$2") ;; *) # unknown option @@ -52,7 +87,10 @@ do done # Testing if directory parametter was used -# If not variable is sets to default value /tmp/repo/resources/pkg/rpm +# If not variable is sets to default value: +# /tmp/repo/resources/pkg/rpm +# or +# /tmp/repo/resources/pkg/deb if test -z "$OFFLINE_REPO_DIR" then OFFLINE_REPO_DIR="/tmp/repo/" @@ -60,21 +98,93 @@ fi # Testing if list parametter was used # If not variable is sets to default value /tmp/offline/data-list -if test -z "$RPM_LIST_DIR" +if test -z "$PCKG_LIST_DIR" then - RPM_LIST_DIR="/tmp/offline/data_list/" + PCKG_LIST_DIR="/tmp/offline/data_list/" +fi +# Testing if additional packages list parametter was used +# If not variable is sets to default value /tmp/additional-lists +if test -z "$PCKG_LIST_DIR" +then + PCKG_LIST_DIR="/tmp/additional-lists/" fi -# Install createrepo package for create repository in folder -# and yum-utils due to yum-config-manager for adding docker repository -yum install createrepo yum-utils -y +case "$distro_type" in + ubuntu) + # Change current working dir + pushd $OFFLINE_REPO_DIR + + # Install dpkg-deb package for create repository in folder + # Install software-properties-common to get add-apt-repository command + # Install apt-transport-https, ca-certificates, curl and gnupg-agent allowing apt to use a repository over HTTPS + apt-get update -y + apt-get install dpkg-dev apt-transport-https ca-certificates curl gnupg-agent software-properties-common -y + + # Add Docker's official GPG key: + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + apt-key fingerprint 0EBFCD88 + + # Add docker repository + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + # Temp fix of known bug + # https://bugs.launchpad.net/ubuntu/+source/aptitude/+bug/1543280 + chown _apt $OFFLINE_REPO_DIR + + # Download all packages from onap_deb.list via apt-get to repository folder + for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}');do apt-get download $i -y; done + for i in $(cat ${PCKG_LIST_DIR}onap_deb.list | awk '{print $1}'); + do + for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/); + do apt-get download $depends -y; + done; + done + + # Download all packages with dependecies from all additional packages lists via apt-get to repository folder + if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then + for list in ${ADDITIONAL_LISTS[@]} + do + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do apt-get download $i -y; done + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}'); + do + for depends in $(apt-cache depends $i | grep -E 'Depends' | cut -d ':' -f 2,3 | sed -e s/'<'/''/ -e s/'>'/''/); + do apt-get download $depends -y; + done; + done + done + fi + + # In repository folder create gz package with deb packages + dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz + ;; + + rhel) + # Install createrepo package for create repository in folder, + # yum-utils due to yum-config-manager for adding docker repository + # and epel-release for additional packages (like jq etc.) + yum install createrepo yum-utils epel-release -y + + # Add official docker repository + yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/ + + # Download all packages from onap_rpm.list via yumdownloader to repository folder + for i in $(cat ${PCKG_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done -# Add official docker repository -yum-config-manager --add-repo=https://download.docker.com/linux/centos/7/x86_64/stable/ + # Download all packages from all additional packages lists via apt-get to repository folder + if ! [ ${#ADDITIONAL_LISTS[@]} -eq 0 ]; then + for list in ${ADDITIONAL_LISTS[@]} + do + for i in $(cat ${ADD_LIST_DIR}$list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done + done + fi -# Download all packages from onap_rpm.list via yumdownloader to repository folder -for i in $(cat ${RPM_LIST_DIR}onap_rpm.list | awk '{print $1}');do yumdownloader --resolve --downloadonly --destdir=${OFFLINE_REPO_DIR} $i -y; done + # In repository folder create repositor + createrepo $OFFLINE_REPO_DIR + ;; -# In repository folder create repository -createrepo $OFFLINE_REPO_DIR + *) + echo "Unknown type of linux distribution." + exit 1 + ;; +esac diff --git a/build/package.py b/build/package.py index c0ca47a5..9e37d4bf 100755 --- a/build/package.py +++ b/build/package.py @@ -28,6 +28,7 @@ import glob import json import sys import os +import hashlib import tarfile import git @@ -82,20 +83,43 @@ def create_package_info_file(output_file, repository_list, tag, metadata): build_info = { 'Build_info': { 'build_date': datetime.now().strftime('%Y-%m-%d_%H-%M'), - 'Version': tag + 'Version': tag, + 'Packages': {} } } for repository in repository_list: build_info['Build_info'][ repository.config_reader().get_value('remote "origin"', 'url')] = repository.head.commit.hexsha - if len(metadata) != 0: - build_info['Build_info'][metadata[0]] = metadata[1] + if metadata: + for meta in metadata: + build_info['Build_info'].update(meta) with open(output_file, 'w') as outfile: json.dump(build_info, outfile, indent=4) +def add_checksum_info(output_dir): + """ + Add checksum information into package.info file + :param output_dir: directory where are packages + """ + tar_files = ['resources_package.tar', 'aux_package.tar', 'sw_package.tar'] + for tar_file in tar_files: + try: + checksum = hashlib.md5() + with open(os.path.join(output_dir, tar_file), 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + checksum.update(chunk) + with open(os.path.join(output_dir, 'package.info'), 'r') as f: + json_data = json.load(f) + json_data['Build_info']['Packages'].update({tar_file: checksum.hexdigest()}) + with open(os.path.join(output_dir, 'package.info'), 'w') as f: + json.dump(json_data, f, indent=4) + except FileNotFoundError: + pass + + def create_package(tar_content, file_name): """ Creates packages @@ -109,6 +133,20 @@ def create_package(tar_content, file_name): output_tar_file.add(src, dst) +def metadata_validation(param): + """ + Validation of metadata parameters + :param param: parameter to be checked needs to be in format key=value + """ + try: + key, value = param.split('=') + assert (key and value) + return {key: value} + except (ValueError, AssertionError): + msg = "%r is not a valid parameter. Needs to be in format key=value" % param + raise argparse.ArgumentTypeError(msg) + + def build_offline_deliverables(build_version, application_repository_url, application_repository_reference, @@ -224,6 +262,7 @@ def build_offline_deliverables(build_version, aux_package_tar_path = os.path.join(output_dir, 'aux_package.tar') create_package(aux_content, aux_package_tar_path) + add_checksum_info(output_dir) shutil.rmtree(application_dir) @@ -265,8 +304,8 @@ def run_cli(): help='overwrite files in output directory') parser.add_argument('--debug', action='store_true', default=False, help='Turn on debug output') - parser.add_argument('--add-metadata', nargs=2, - help='additional metadata added into package.info, format: key value', default=[]) + parser.add_argument('--add-metadata', nargs="+", type=metadata_validation, + help='additional metadata added into package.info, format: key=value') args = parser.parse_args() if args.debug: diff --git a/build/requirements.txt b/build/requirements.txt index 39544458..441b3fcb 100644 --- a/build/requirements.txt +++ b/build/requirements.txt @@ -1,2 +1,2 @@ docker>=3.7.2 -gitpython==2.1.11 +gitpython==3.1.0 diff --git a/docs/.gitignore b/docs/.gitignore index 19fe1aa0..43ca5b67 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,2 +1,3 @@ -conf.py* -_static +/.tox +/_build/* +/__pycache__/* diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst index 5b2e2486..e2215c11 100644 --- a/docs/BuildGuide.rst +++ b/docs/BuildGuide.rst @@ -61,9 +61,9 @@ Subsequent steps are the same on both platforms: :: # install following packages - yum install -y docker-ce-18.09.5 python-pip git createrepo expect nodejs npm jq + yum install -y docker-ce-18.09.5 git createrepo expect nodejs npm jq - # install Python 3 (download scripts don't support Python 2 anymore) + # install Python 3 yum install -y python36 python36-pip # docker daemon must be running on host @@ -111,12 +111,12 @@ Part 2. Download artifacts for offline installer .. note:: Skip this step if you have already all necessary resources and continue with Part 3. Populate local nexus -A RPM repository containing packages to be installed on all nodes needs to be created: +Repository containing packages to be installed on all nodes needs to be created: :: - # run the docker container with -d parameter for destination directory with RPM packages - ./offline-installer/build/create_repo.sh -d $(pwd) + # run the docker container with -d parameter for destination directory with RPM packages and optionally use -t parameter for target platform. Supported target platforms are centos|rhel|ubuntu. If -t parameter is not given, default platform is based on host platform where script is running. + ./offline-installer/build/create_repo.sh -d $(pwd) -t centos|rhel|ubuntu .. note:: If script fails due to permissions issue, it could be a problem with SeLinux. It can be fixed by running: :: diff --git a/docs/_static/css/ribbon.css b/docs/_static/css/ribbon.css new file mode 100644 index 00000000..6008cb1a --- /dev/null +++ b/docs/_static/css/ribbon.css @@ -0,0 +1,63 @@ +.ribbon { + z-index: 1000; + background-color: #a00; + overflow: hidden; + white-space: nowrap; + position: fixed; + top: 25px; + right: -50px; + -webkit-transform: rotate(45deg); + -moz-transform: rotate(45deg); + -ms-transform: rotate(45deg); + -o-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 0 10px #888; + -moz-box-shadow: 0 0 10px #888; + box-shadow: 0 0 10px #888; + +} + +.ribbon a { + border: 1px solid #faa; + color: #fff; + display: block; + font: bold 81.25% 'Helvetica Neue', Helvetica, Arial, sans-serif; + margin: 1px 0; + padding: 10px 50px; + text-align: center; + text-decoration: none; + text-shadow: 0 0 5px #444; + transition: 0.5s; +} + +.ribbon a:hover { + background: #c11; + color: #fff; +} + + +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td, .wy-table-responsive table th { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} + +@media screen and (max-width: 767px) { + .wy-table-responsive table td { + white-space: nowrap; + } +} + +/* fix width of the screen */ + +.wy-nav-content { + max-width: none; +} diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico Binary files differnew file mode 100755 index 00000000..cb712ebd --- /dev/null +++ b/docs/_static/favicon.ico diff --git a/docs/_static/logo_onap_2017.png b/docs/_static/logo_onap_2017.png Binary files differnew file mode 100644 index 00000000..5d064f43 --- /dev/null +++ b/docs/_static/logo_onap_2017.png diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..8f40e8b8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,15 @@ +from docs_conf.conf import * + +branch = 'latest' +master_doc = 'index' + +linkcheck_ignore = [ + 'http://localhost', +] + +intersphinx_mapping = {} + +html_last_updated_fmt = '%d-%b-%y %H:%M' + +def setup(app): + app.add_stylesheet("css/ribbon_onap.css") diff --git a/docs/conf.yaml b/docs/conf.yaml new file mode 100644 index 00000000..ab592813 --- /dev/null +++ b/docs/conf.yaml @@ -0,0 +1,7 @@ +--- +project_cfg: onap +project: onap + +# Change this to ReleaseBranchName to modify the header +default-version: latest +# diff --git a/docs/index.rst b/docs/index.rst index 4f50860b..8d187225 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,5 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. _master_index: OOM offline-installer ===================== diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 00000000..b3188ddd --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,15 @@ +tox +Sphinx +doc8 +docutils +setuptools +six +sphinx_rtd_theme>=0.4.3 +sphinxcontrib-blockdiag +sphinxcontrib-needs>=0.2.3 +sphinxcontrib-nwdiag +sphinxcontrib-seqdiag +sphinxcontrib-swaggerdoc +sphinxcontrib-plantuml +sphinx_bootstrap_theme +lfdocs-conf diff --git a/docs/tox.ini b/docs/tox.ini new file mode 100644 index 00000000..edac8c35 --- /dev/null +++ b/docs/tox.ini @@ -0,0 +1,22 @@ +[tox] +minversion = 1.6 +envlist = docs, +skipsdist = true + +[testenv:docs] +basepython = python3 +deps = -r{toxinidir}/requirements-docs.txt +commands = + sphinx-build -b html -n -d {envtmpdir}/doctrees ./ {toxinidir}/_build/html + echo "Generated docs available in {toxinidir}/_build/html" +whitelist_externals = + echo + git + sh + +[testenv:docs-linkcheck] +basepython = python3 +#deps = -r{toxinidir}/requirements-docs.txt +commands = echo "Link Checking not enforced" +#commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./ {toxinidir}/_build/linkcheck +whitelist_externals = echo diff --git a/tools/cicdansible/heat/config.yaml b/tools/cicdansible/heat/config.yaml index e1f0309f..0521d72e 100644 --- a/tools/cicdansible/heat/config.yaml +++ b/tools/cicdansible/heat/config.yaml @@ -6,5 +6,5 @@ output: { all: "/dev/console" } #Initialization. runcmd: - | - set -efxu -o pipefail + set -efxu %{NOTIFY_COMMAND} --data-binary '{"status": "SUCCESS", "reason": "instance started successfully"}' diff --git a/tools/cicdansible/roles/install/tasks/install.yml b/tools/cicdansible/roles/install/tasks/install.yml index 529e2acf..5c4bcd81 100644 --- a/tools/cicdansible/roles/install/tasks/install.yml +++ b/tools/cicdansible/roles/install/tasks/install.yml @@ -14,7 +14,7 @@ unarchive: src: "resources/{{ hostvars[groups['resources'][0]].resources_sw_filename }}" dest: "{{ installer_deploy_path }}" -#Generate ansible inventory and extra vars. +#Generate ansible inventory and extra vars - name: "Generate ansible inventory for installer" template: src: inventory.yml.j2 diff --git a/tools/cicdansible/roles/install/templates/inventory.yml.j2 b/tools/cicdansible/roles/install/templates/inventory.yml.j2 index faec5903..9f7e08f8 100644 --- a/tools/cicdansible/roles/install/templates/inventory.yml.j2 +++ b/tools/cicdansible/roles/install/templates/inventory.yml.j2 @@ -1,5 +1,8 @@ all: vars: +{% if hostvars['infra'].ansible_distribution in ["Debian","Ubuntu"] %} + ansible_python_interpreter: "/usr/bin/python3" +{% endif %} ansible_ssh_private_key_file: /root/.ssh/id_rsa ansible_ssh_common_args: "-o StrictHostKeyChecking=no" children: diff --git a/tools/cicdansible/roles/setup_openstack_infrastructure/tasks/configure/volume.yml b/tools/cicdansible/roles/setup_openstack_infrastructure/tasks/configure/volume.yml index 8c553850..568b7202 100644 --- a/tools/cicdansible/roles/setup_openstack_infrastructure/tasks/configure/volume.yml +++ b/tools/cicdansible/roles/setup_openstack_infrastructure/tasks/configure/volume.yml @@ -7,7 +7,7 @@ partition_path: "{{ volume_path }}-part1" - name: "Wait for volume" #We do not do it normally, because we want to trigger udev (workaround for some bugs). - shell: "udevadm trigger && udevadm settle && [[ -b {{ volume_path }} ]]" + shell: "udevadm trigger && udevadm settle && [ -b {{ volume_path }} ]" register: result retries: 30 delay: 10 |
