noheat: Upgrades, refactor, deploy Devstack

* Revert "Enable VID" (this reverts commit 2223bfaffa8fe5fb90c578ec71a035f001b9ce22)
* make sure ca-certificates is current
* add ability to configure Docker daemon
* upgrade Ubuntu to 20.04
* update tools, dependencies, Ansible Galaxy collections, Docker, RKE, k8s & charts
* make Prometheus & MetalLB optional
* deploy Istio
* use copy instead of ansible.posix.synchronize
* Allow to set IP address pools in OpenStack
* fix ansible-lint issues
* split big tasks files into smaller ones
* migrate to ansible-core
* add 'make onap' task
* add Devstack deployment
* fix NFS exports
* add Strimzi
* install Docker on operator
* override Nexus only on hosts that need it
* ability to set override file (sm-onap is default)
* ability to set ONAP (OOM) branch
* ability to set Devstack version
* add playbook to deploy everything
* describe how to run playbooks
* save operator0 access information
* install Galaxy deps with dedicated module

Issue-ID: INT-1601
Signed-off-by: Maciej Wereski <m.wereski@partner.samsung.com>
Change-Id: I88cfeeaf281c175340b63909983251ecd62eeb00

21 files changed, 181 insertions, 48 deletions

diff --git a/deployment/noheat/infra-openstack/README.rst b/deployment/noheat/infra-openstack/README.rst
index 1ac9539f5..c48dfa7f2 100644
--- a/deployment/noheat/infra-openstack/README.rst
+++ b/deployment/noheat/infra-openstack/README.rst
@@ -22,13 +22,8 @@ Configuration
 Dependencies
 ~~~~~~~~~~~~
 
-- Ansible: tested on 2.9.9 (using Python 3.5.2)
-    - Collections
-        - community.crypto: tested on 1.3.0
-        - ansible.posix: tested on 1.1.1
-    - Roles
-        - geerlingguy.ansible: tested on 2.1.0
-- openstacksdk_: tested on 0.46.0 (using Python 3.5.2)
+Tested on Python 3.8.10. Required Python dependencies can be found in ``../requirements.txt``.
+Required Ansible roles and collections can be found in ``../requirements.yml``
 
 .. _openstacksdk: https://pypi.org/project/openstacksdk
 
diff --git a/deployment/noheat/infra-openstack/ansible/create.yml b/deployment/noheat/infra-openstack/ansible/create.yml
index 825bee3a4..73830663c 100644
--- a/deployment/noheat/infra-openstack/ansible/create.yml
+++ b/deployment/noheat/infra-openstack/ansible/create.yml
@@ -10,6 +10,21 @@
     - role: create_hosts
       hosts: "{{ operation.hosts }}"
       operator_key: "dummy"
+  tasks:
+    - name: Get operator Openstack info
+      openstack.cloud.server_info:
+        server: "operator0"
+      register: operator_info
+    - name: Create directory for artifacts
+      ansible.builtin.file:
+        name: "artifacts"
+        state: directory
+        mode: '0755'
+    - name: Save operator access information
+      ansible.builtin.copy:
+        content: "{{ operator_info['openstack_servers'][0]['public_v4'] }},{{ image['user'] }},~/.ssh/{{ keypair['key']['name'] }}"
+        dest: "artifacts/operator.csv"
+        mode: "0644"
 - name: Create cluster operator access keypair
   hosts: "operator0"
   gather_facts: False
@@ -21,10 +36,18 @@
         path: "~/.ssh/{{ keypair.name }}"
       register: key
     - name: Add operator0 public key to it's authorized keys
-      authorized_key:
+      ansible.posix.authorized_key:
         key: "{{ key['public_key'] }}"
         state: present
         user: "{{ ansible_user }}"
+- name: Create OpenStack instances
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  roles:
+    - role: create_hosts
+      hosts: "{{ openstack.hosts }}"
+      operator_key: "{{ hostvars['operator0']['key']['public_key'] }}"
 - name: Create cluster instances
   hosts: localhost
   connection: local
@@ -35,9 +58,8 @@
       operator_key: "{{ hostvars['operator0']['key']['public_key'] }}"
 - name: Create cluster operator access information
   hosts: "operator0"
-  roles:
-    - role: geerlingguy.ansible
-      become: yes
+  vars_files:
+    - ../../common-vars.yml
   tasks:
     - name: Add cluster hostnames to /etc/hosts file
       lineinfile:
@@ -52,25 +74,63 @@
       vars:
         hosts: "{{ lookup('dict', hostvars['localhost']['hosts_dict']) }}"
     - name: Push in-cluster deployment stage description to the next Ansible control host
-      ansible.posix.synchronize:
+      copy:
         src: ../../cluster-rke
         dest: ~/deploy
-    - name: Install python dependencies
-      become: yes
-      package:
-        name:
-          - python3-pip
-          - python3-setuptools
-        state: present
-    - name: Install community.kubernetes.k8s Ansible collection dependencies
-      pip:
-        name:
-          - openshift
-          - pyyaml
-          - kubernetes
-        executable: pip3
-      become: yes
-    - name: Add Ansible collection dependencies
-      command: "ansible-galaxy collection install ansible.posix"
-    - name: Add community.kubernetes Ansible collection
-      command: "ansible-galaxy collection install community.kubernetes"
+    - name: Push Devstack deployment stage description to the next Ansible control host
+      copy:
+        src: ../../devstack
+        dest: ~/
+    - name: Push common variables to the next Ansible control host
+      copy:
+        src: ../../common-vars.yml
+        dest: ~/
+    - name: Push Devstack vars to the next Ansible control host (for Devstack stage)
+      template:
+        src: "templates/openstack.yml.j2"
+        dest: ~/devstack/ansible/group_vars/all/openstack.yml
+        mode: '0644'
+    - name: Push Devstack vars to the next Ansible control host (for cluster-rke stage)
+      template:
+        src: "templates/openstack.yml.j2"
+        dest: ~/deploy/cluster-rke/ansible/group_vars/all/openstack.yml
+        mode: '0644'
+    - name: Create Devstack config directory
+      file:
+        path: ~/.config/openstack/
+        state: directory
+        mode: '0755'
+    - name: Generate Devstack clouds.yml file
+      template:
+        src: "templates/clouds.yaml.j2"
+        dest: ~/.config/openstack/clouds.yml
+        mode: '0644'
+    - block:
+      - name: Install python dependencies
+        become: yes
+        apt:
+          name:
+            - python3-pip
+            - python3-setuptools
+            - default-jdk-headless
+          state: present
+          update_cache: true
+      - name: Install community.kubernetes.k8s Ansible collection dependencies
+        pip:
+          name:
+            - ansible-core==2.13.5
+            - openshift==0.13.1
+            - pyyaml==6.0
+            # Major version of Python k8s libraty matches minor version of k8s.
+            - kubernetes~={{ k8s_version | regex_search("[^^.][0-9]+[^$]") ~ "0" }}
+          executable: pip3
+        become: yes
+      - name: Copy ansible-galaxy requirements file
+        copy:
+          src: operator-requirements.yml
+          dest: ~/requirements.yml
+          mode: '0444'
+      - name: Install ansible-galaxy collections
+        community.general.ansible_galaxy_install:
+          requirements_file: ~/requirements.yml
+          type: both
diff --git a/deployment/noheat/infra-openstack/ansible/destroy.yml b/deployment/noheat/infra-openstack/ansible/destroy.yml
index ff9d5fccf..1564e3088 100644
--- a/deployment/noheat/infra-openstack/ansible/destroy.yml
+++ b/deployment/noheat/infra-openstack/ansible/destroy.yml
@@ -8,6 +8,8 @@
       hosts: "{{ cluster.hosts }}"
     - role: destroy_hosts
       hosts: "{{ operation.hosts }}"
+    - role: destroy_hosts
+      hosts: "{{ openstack.hosts }}"
     - destroy_keypair
     - destroy_network
     - destroy_securitygroup
diff --git a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml b/deployment/noheat/infra-openstack/ansible/group_vars/all.yml
deleted file mode 120000
index e4e1e1f50..000000000
--- a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml
+++ /dev/null
@@ -1 +0,0 @@
-all.yml.sample
\ No newline at end of file
diff --git a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sample b/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sample
index 1b03b06d5..541e15279 100644
--- a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sample
+++ b/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sample
@@ -18,9 +18,23 @@ securitygroup:
     - "192.168.1.0/24"
 
 image:
-  name: &image_name "Ubuntu_18.04"
+  name: &image_name "Ubuntu_20.04"
   user: "ubuntu"
 
+openstack:
+  name: "vnf0"
+  inventory: "~/inventory.ini"
+  hosts:
+    - name: "openstack0"
+      image: *image_name
+      flavor: "m1.large"
+      keypair: *keypair_name
+      network: *network_name
+      securitygroup: *securitygroup_name
+      boot_from_volume: true
+      terminate_volume: true
+      volume_size: 100
+
 operation:
   name: "operation0"
   inventory: "~/inventory.ini"
diff --git a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sm-onap b/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sm-onap
index cb5168c26..9223ea591 100644
--- a/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sm-onap
+++ b/deployment/noheat/infra-openstack/ansible/group_vars/all.yml.sm-onap
@@ -14,9 +14,22 @@ securitygroup:
     - "192.168.1.0/24"
 
 image:
-  name: &image_name "Ubuntu_18.04"
+  name: &image_name "Ubuntu_20.04"
   user: "ubuntu"
 
+openstack:
+  name: "vnf0"
+  inventory: "~/inventory.ini"
+  hosts:
+    - name: "openstack0"
+      image: *image_name
+      flavor: "m1.xlarge"
+      keypair: *keypair_name
+      network: *network_name
+      auto_ip: false
+      securitygroup: *securitygroup_name
+      volume_size: 140
+
 operation:
   name: "operation0"
   inventory: "~/inventory.ini"
diff --git a/deployment/noheat/infra-openstack/ansible/group_vars/all/all.yml b/deployment/noheat/infra-openstack/ansible/group_vars/all/all.yml
new file mode 120000
index 000000000..854839817
--- /dev/null
+++ b/deployment/noheat/infra-openstack/ansible/group_vars/all/all.yml
@@ -0,0 +1 @@
+../all.yml.sample
\ No newline at end of file
diff --git a/deployment/noheat/infra-openstack/ansible/group_vars/all/openstack.yml b/deployment/noheat/infra-openstack/ansible/group_vars/all/openstack.yml
new file mode 100644
index 000000000..63ed1b081
--- /dev/null
+++ b/deployment/noheat/infra-openstack/ansible/group_vars/all/openstack.yml
@@ -0,0 +1,6 @@
+---
+openstack_username: "admin"
+openstack_domain: "Default"
+openstack_passwd: "secret"
+openstack_region: "RegionOne"
+openstack_tenant: "admin"
diff --git a/deployment/noheat/infra-openstack/ansible/operator-requirements.yml b/deployment/noheat/infra-openstack/ansible/operator-requirements.yml
new file mode 100644
index 000000000..08b7eeea1
--- /dev/null
+++ b/deployment/noheat/infra-openstack/ansible/operator-requirements.yml
@@ -0,0 +1,8 @@
+---
+collections:
+  - name: ansible.posix
+    version: 1.4.0
+  - name: kubernetes.core
+    version: 2.3.2
+  - name: community.general
+    version: 5.7.0
diff --git a/deployment/noheat/infra-openstack/ansible/roles/create_hosts/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/create_hosts/tasks/main.yml
index 731bca04f..933b2f526 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/create_hosts/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/create_hosts/tasks/main.yml
@@ -1,3 +1,5 @@
 ---
-- include: create_host.yml host={{ item }}
+- include_tasks: create_host.yml
   loop: "{{ hosts }}"
+  loop_control:
+    loop_var: host
diff --git a/deployment/noheat/infra-openstack/ansible/roles/create_keypair/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/create_keypair/tasks/main.yml
index a330875d8..8a7c72092 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/create_keypair/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/create_keypair/tasks/main.yml
@@ -5,11 +5,11 @@
   register: keypair
 
 - name: Create local public key
-  local_action:
-    module: copy
+  copy:
     content: "{{ keypair.key.public_key }}"
     dest: "~/.ssh/{{ keypair.key.name }}.pub"
     mode: 0600
+  delegate_to: localhost
 
 - name: Check if local private key exists
   stat:
@@ -17,9 +17,9 @@
   register: local_private_key
 
 - name: Create local private key
-  local_action:
-    module: copy
+  copy:
     content: "{{ keypair.key.private_key }}"
     dest: "~/.ssh/{{ keypair.key.name }}"
     mode: 0600
-  when: local_private_key.stat.exists == False
+  delegate_to: localhost
+  when: not local_private_key.stat.exists
diff --git a/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/create_network.yml b/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/create_network.yml
index 81d8caae1..3e22ee6ce 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/create_network.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/create_network.yml
@@ -15,6 +15,8 @@
     network_name: "{{ net.name }}"
     cidr: "{{ net.cidr }}"
     dns_nameservers: "{{ dns_ips if dns_ips is defined else omit }}"
+    allocation_pool_start: '{{ net.allocation_pool_start | default("") }}'
+    allocation_pool_end: '{{ net.allocation_pool_end | default ("") }}'
     state: present
 
 - name: "Create {{ net.name }} router"
diff --git a/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/main.yml
index 5e3ef67f5..cce6f790b 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/create_network/tasks/main.yml
@@ -1,4 +1,6 @@
 ---
-- include: create_network.yml net={{ item }}
+- include_tasks: create_network.yml
   loop:
     - "{{ network }}"
+  loop_control:
+    loop_var: net
diff --git a/deployment/noheat/infra-openstack/ansible/roles/create_securitygroup/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/create_securitygroup/tasks/main.yml
index d04b72c34..872988032 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/create_securitygroup/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/create_securitygroup/tasks/main.yml
@@ -1,5 +1,5 @@
 ---
-- include: create_securitygroup.yml
+- include_tasks: create_securitygroup.yml
   loop:
     - "{{ securitygroup }}"
   loop_control:
diff --git a/deployment/noheat/infra-openstack/ansible/roles/destroy_hosts/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/destroy_hosts/tasks/main.yml
index 5ce130d44..1dd5c7224 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/destroy_hosts/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/destroy_hosts/tasks/main.yml
@@ -1,3 +1,5 @@
 ---
-- include: destroy_host.yml host={{ item }}
+- include_tasks: destroy_host.yml
   loop: "{{ hosts }}"
+  loop_control:
+    loop_var: host
diff --git a/deployment/noheat/infra-openstack/ansible/roles/destroy_network/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/destroy_network/tasks/main.yml
index e52dcbdb8..1d84ab62a 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/destroy_network/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/destroy_network/tasks/main.yml
@@ -1,4 +1,6 @@
 ---
-- include: destroy_network.yml net={{ item }}
+- include_tasks: destroy_network.yml
   loop:
     - "{{ network }}"
+  loop_control:
+    loop_var: net
diff --git a/deployment/noheat/infra-openstack/ansible/roles/destroy_securitygroup/tasks/main.yml b/deployment/noheat/infra-openstack/ansible/roles/destroy_securitygroup/tasks/main.yml
index de098afec..8142e8070 100644
--- a/deployment/noheat/infra-openstack/ansible/roles/destroy_securitygroup/tasks/main.yml
+++ b/deployment/noheat/infra-openstack/ansible/roles/destroy_securitygroup/tasks/main.yml
@@ -1,4 +1,6 @@
 ---
-- include: destroy_securitygroup.yml secgrp={{ item }}
+- include_tasks: destroy_securitygroup.yml
   loop:
     - "{{ securitygroup }}"
+  loop_control:
+    loop_var: secgrp
diff --git a/deployment/noheat/infra-openstack/ansible/templates/clouds.yaml.j2 b/deployment/noheat/infra-openstack/ansible/templates/clouds.yaml.j2
new file mode 100644
index 000000000..afbbc8738
--- /dev/null
+++ b/deployment/noheat/infra-openstack/ansible/templates/clouds.yaml.j2
@@ -0,0 +1,11 @@
+clouds:
+  openstack:
+    auth:
+      auth_url: "https://{{ hostvars['localhost']['hosts_dict']['openstack0'] }}:5000/v3"
+      project_name: "{{ openstack_tenant }}""
+      username: "{{ openstack_username }}"
+      user_domain_name: "{{ openstack_domain }}"
+      password: "{{ openstack_passwd }}"
+    region_name: "{{ openstack_region }}"
+    interface: "public"
+    identity_api_version: 3
diff --git a/deployment/noheat/infra-openstack/ansible/templates/inventory.ini.j2 b/deployment/noheat/infra-openstack/ansible/templates/inventory.ini.j2
index 15bb7caf9..79da2c603 100644
--- a/deployment/noheat/infra-openstack/ansible/templates/inventory.ini.j2
+++ b/deployment/noheat/infra-openstack/ansible/templates/inventory.ini.j2
@@ -3,6 +3,13 @@
 {{ item.key }} ansible_host={{ item.value }}
 {% endfor %}
 
+[openstack]
+{% for item in hosts %}
+{% if "openstack" in item.key %}
+{{ item.key }}
+{% endif %}
+{% endfor %}
+
 [operator]
 {% for item in hosts %}
 {% if "operator" in item.key %}
diff --git a/deployment/noheat/infra-openstack/ansible/templates/openstack.yml.j2 b/deployment/noheat/infra-openstack/ansible/templates/openstack.yml.j2
new file mode 100644
index 000000000..25233abca
--- /dev/null
+++ b/deployment/noheat/infra-openstack/ansible/templates/openstack.yml.j2
@@ -0,0 +1,5 @@
+---
+openstack_username: "{{ openstack_username }}"
+openstack_passwd: "{{ openstack_passwd }}"
+openstack_region: "{{ openstack_region }}"
+openstack_tenant: "{{ openstack_tenant }}"
diff --git a/deployment/noheat/infra-openstack/vagrant/Vagrantfile b/deployment/noheat/infra-openstack/vagrant/Vagrantfile
index 8acdf1442..ed1a3d076 100644
--- a/deployment/noheat/infra-openstack/vagrant/Vagrantfile
+++ b/deployment/noheat/infra-openstack/vagrant/Vagrantfile
@@ -11,15 +11,15 @@ os_clouds_dir = "${HOME}/.config/openstack"
 os_clouds_config = "#{os_clouds_dir}/clouds.yaml"
 os_admin = "admin"
 os_user = "demo"
-image_url = "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img"
-image_name = "Ubuntu_18.04"
+image_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img"
+image_name = "Ubuntu_20.04"
 
 vm_cpu = 1
 vm_cpus = 4
 vm_memory = 1 * 1024
 vm_memory_os = 8 * 1024
 vm_disk = 32
-vm_box = "generic/ubuntu1804"
+vm_box = "generic/ubuntu2004"
 
 operation = {
   name: 'operator',