Add molecule tests for rke role

This change adds molecule tests for the rke role, and
modifies the rke role itself to be more idempotent/to
pass linter.
Note that this molecule test case uses a separate
role to install docker in containers, that runs
docker daemon inside of them instead of using host docker.

Issue-ID: OOM-1778
Change-Id: I875f3ff2ab961e5428acee5a02287a8d2d6e9969
Signed-off-by: Michal Zegan <m.zegan@samsung.com>

10 files changed, 261 insertions, 0 deletions

diff --git a/ansible/roles/rke/.yamllint b/ansible/roles/rke/.yamllint
new file mode 100644
index 00000000..ad0be760
--- /dev/null
+++ b/ansible/roles/rke/.yamllint
@@ -0,0 +1,11 @@
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  line-length: disable
+  truthy: disable
diff --git a/ansible/roles/rke/molecule/default/destroy.yml b/ansible/roles/rke/molecule/default/destroy.yml
new file mode 100644
index 00000000..591da82e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/destroy.yml
@@ -0,0 +1,34 @@
+---
+- name: Destroy
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+  tasks:
+    - name: Destroy molecule instance(s)
+      docker_container:
+        name: "{{ item.name }}"
+        docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        state: absent
+        force_kill: "{{ item.force_kill | default(true) }}"
+        # Modification: we want to clean up old volumes.
+        keep_volumes: false
+      register: server
+      with_items: "{{ molecule_yml.platforms }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for instance(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: docker_jobs
+      until: docker_jobs.finished
+      retries: 300
+      with_items: "{{ server.results }}"
+
+    - name: Delete docker network(s)
+      docker_network:
+        name: "{{ item }}"
+        docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+        state: absent
+      with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks }}"
diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml
new file mode 100644
index 00000000..e8e5ad76
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/molecule.yml
@@ -0,0 +1,78 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: infrastructure-server
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: true
+    privileged: true
+    override_command: false
+    restart_policy: unless-stopped
+    volumes:
+      - /var/lib/kubelet
+      - /var/lib/docker
+    env:
+      container: docker
+    groups:
+      - infrastructure
+      - kubernetes-control-plane
+    networks:
+      - name: rke
+    purge_networks: true
+
+  - name: kubernetes-node-1
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: true
+    privileged: true
+    override_command: false
+    restart_policy: unless-stopped
+    env:
+      container: docker
+    volumes:
+      - /var/lib/kubelet
+      - /var/lib/docker
+    groups:
+      - kubernetes
+    networks:
+      - name: rke
+    purge_networks: true
+
+  - name: kubernetes-node-2
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: true
+    privileged: true
+    override_command: false
+    restart_policy: unless-stopped
+    env:
+      container: docker
+    volumes:
+      - /var/lib/kubelet
+      - /var/lib/docker
+    groups:
+      - kubernetes
+    networks:
+      - name: rke
+    purge_networks: true
+
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../test/roles
+    ANSIBLE_LIBRARY: ../../../../library
+  inventory:
+    links:
+      group_vars: ../../../../group_vars
+  options:
+    e: "app_data_path=/opt/onap"
+  lint:
+    name: ansible-lint
+scenario:
+  name: default
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
diff --git a/ansible/roles/rke/molecule/default/playbook.yml b/ansible/roles/rke/molecule/default/playbook.yml
new file mode 100644
index 00000000..09dbfb8e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/playbook.yml
@@ -0,0 +1,30 @@
+---
+- name: "Set cluster_ip"
+  hosts: all
+  tasks:
+    - name: "Set cluster_ip fact"
+      set_fact:
+        cluster_ip: "{{ ansible_default_ipv4.address }}"
+
+- name: Configure kubernetes cluster (RKE)
+  hosts: infrastructure
+  roles:
+    - role: rke
+      vars:
+        mode: config
+
+- name: Prepare kubernetes nodes (RKE)
+  hosts:
+    - kubernetes
+    - kubernetes-control-plane
+  roles:
+    - role: rke
+      vars:
+        mode: node
+
+- name: Deploy kubernetes cluster (RKE)
+  hosts: infrastructure
+  roles:
+    - role: rke
+      vars:
+        mode: deploy
diff --git a/ansible/roles/rke/molecule/default/prepare.yml b/ansible/roles/rke/molecule/default/prepare.yml
new file mode 100644
index 00000000..6bad2b80
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/prepare.yml
@@ -0,0 +1,15 @@
+---
+- name: "Prepare hosts"
+  hosts: all
+  roles:
+    - role: prepare-rke
+      vars:
+        mode: all
+    - prepare-docker-dind
+
+- name: "Infra specific preparations"
+  hosts: infrastructure
+  roles:
+    - role: prepare-rke
+      vars:
+        mode: infra
diff --git a/ansible/roles/rke/molecule/default/tests/test_controlplane.py b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
new file mode 100644
index 00000000..0bfbca2d
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
@@ -0,0 +1,14 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts(
+      'kubernetes-control-plane')
+
+
+@pytest.mark.parametrize('container_name', [
+  'kube-apiserver', 'kube-controller-manager', 'kube-scheduler', 'kubelet'])
+def test_container_running(host, container_name):
+    assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/default/tests/test_infrastructure.py b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
new file mode 100644
index 00000000..9ba11d6e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
@@ -0,0 +1,56 @@
+import os
+import pytest
+import json
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('infrastructure')
+
+
+@pytest.mark.parametrize('filename', [
+  '/root/.kube/config',
+  '/opt/onap/cluster/cluster.yml',
+  '/opt/onap/cluster/cluster.rkestate'])
+def test_file_existence(host, filename):
+    assert host.file(filename).exists
+
+
+def test_rke_in_path(host):
+    assert host.find_command('rke') == '/usr/local/bin/rke'
+
+
+def test_rke_version_works(host):
+    # Note that we need to cd to the cluster data dir first, really.
+    assert host.run('cd /opt/onap/cluster && rke version').rc == 0
+
+
+def test_nodes_ready(host):
+    # Retrieve all node names.
+    nodecmdres = host.run('kubectl get nodes -o name')
+    assert nodecmdres.rc == 0
+    nodes = nodecmdres.stdout.split('\n')
+    for node in nodes:
+        assert host.run(
+          'kubectl wait --timeout=0 --for=condition=ready ' + node).rc == 0
+
+
+def test_pods_ready(host):
+    # Retrieve all pods from all namespaces.
+    # Because we need pod and namespace name, we get full json representation.
+    podcmdres = host.run('kubectl get pods --all-namespaces -o json')
+    assert podcmdres.rc == 0
+    pods = json.loads(podcmdres.stdout)['items']
+    for pod in pods:
+        # Each pod may be either created by a job or not.
+        # In job case they should already be completed
+        # when we are here so we ignore them.
+        namespace = pod['metadata']['namespace']
+        podname = pod['metadata']['name']
+        condition = 'Ready'
+        if len(pod['metadata']['ownerReferences']) == 1 and pod[
+          'metadata']['ownerReferences'][0]['kind'] == 'Job':
+            continue
+        assert host.run(
+          'kubectl wait --timeout=120s --for=condition=' + condition + ' -n ' +
+          namespace + ' pods/' + podname).rc == 0
diff --git a/ansible/roles/rke/molecule/default/tests/test_kubernetes.py b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
new file mode 100644
index 00000000..887494fa
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
@@ -0,0 +1,13 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes')
+
+
+@pytest.mark.parametrize('container_name', [
+  'etcd', 'kubelet', 'kube-proxy'])
+def test_container_running(host, container_name):
+    assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/tasks/rke_config.yml b/ansible/roles/rke/tasks/rke_config.yml
index 4112e107..9dc0d8c6 100644
--- a/ansible/roles/rke/tasks/rke_config.yml
+++ b/ansible/roles/rke/tasks/rke_config.yml
@@ -37,6 +37,7 @@
   template:
     src: cluster.yml.j2
     dest: "{{ cluster_config_dir }}/cluster.yml"
+  register: cluster_yml
 
 - name: Prepare rke addon manifest (dashboard)
   template:
diff --git a/ansible/roles/rke/tasks/rke_deploy.yml b/ansible/roles/rke/tasks/rke_deploy.yml
index 9983d08a..7b36f55d 100644
--- a/ansible/roles/rke/tasks/rke_deploy.yml
+++ b/ansible/roles/rke/tasks/rke_deploy.yml
@@ -1,8 +1,17 @@
 ---
+- name: "Check if rke is deployed"
+  command: "rke version"
+  args:
+    chdir: "{{ cluster_config_dir }}"
+  failed_when: false
+  changed_when: false
+  register: rke_deployed
+
 - name: Run rke up
   command: "{{ rke_bin_dir }}/rke up --config cluster.yml"
   args:
     chdir: "{{ cluster_config_dir }}"
+  when: rke_deployed.rc != 0 or cluster_yml.changed  # noqa 503
 
 - name: Ensure .kube directory is present
   file: