summaryrefslogtreecommitdiffstats
path: root/ansible/roles/rke
diff options
context:
space:
mode:
authorMichal Zegan <m.zegan@samsung.com>2019-04-30 14:52:13 +0200
committerMichal Zegan <m.zegan@samsung.com>2019-06-03 14:49:32 +0200
commit1ec19b0598382aa45258ed630ee26cba4fb3a7e5 (patch)
tree795a6486f4d40a1890a79e65a0afad6397c44d3e /ansible/roles/rke
parent82052ddde7b2896c05c7b930d8e09d582025f0b3 (diff)
Add molecule tests for rke role
This change adds molecule tests for the rke role, and modifies the rke role itself to be more idempotent/to pass linter. Note that this molecule test case uses a separate role to install docker in containers, that runs docker daemon inside of them instead of using host docker. Issue-ID: OOM-1778 Change-Id: I875f3ff2ab961e5428acee5a02287a8d2d6e9969 Signed-off-by: Michal Zegan <m.zegan@samsung.com>
Diffstat (limited to 'ansible/roles/rke')
-rw-r--r--ansible/roles/rke/.yamllint11
-rw-r--r--ansible/roles/rke/molecule/default/destroy.yml34
-rw-r--r--ansible/roles/rke/molecule/default/molecule.yml78
-rw-r--r--ansible/roles/rke/molecule/default/playbook.yml30
-rw-r--r--ansible/roles/rke/molecule/default/prepare.yml15
-rw-r--r--ansible/roles/rke/molecule/default/tests/test_controlplane.py14
-rw-r--r--ansible/roles/rke/molecule/default/tests/test_infrastructure.py56
-rw-r--r--ansible/roles/rke/molecule/default/tests/test_kubernetes.py13
-rw-r--r--ansible/roles/rke/tasks/rke_config.yml1
-rw-r--r--ansible/roles/rke/tasks/rke_deploy.yml9
10 files changed, 261 insertions, 0 deletions
diff --git a/ansible/roles/rke/.yamllint b/ansible/roles/rke/.yamllint
new file mode 100644
index 00000000..ad0be760
--- /dev/null
+++ b/ansible/roles/rke/.yamllint
@@ -0,0 +1,11 @@
+extends: default
+
+rules:
+ braces:
+ max-spaces-inside: 1
+ level: error
+ brackets:
+ max-spaces-inside: 1
+ level: error
+ line-length: disable
+ truthy: disable
diff --git a/ansible/roles/rke/molecule/default/destroy.yml b/ansible/roles/rke/molecule/default/destroy.yml
new file mode 100644
index 00000000..591da82e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/destroy.yml
@@ -0,0 +1,34 @@
+---
+- name: Destroy
+ hosts: localhost
+ connection: local
+ gather_facts: false
+ no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+ tasks:
+ - name: Destroy molecule instance(s)
+ docker_container:
+ name: "{{ item.name }}"
+ docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+ state: absent
+ force_kill: "{{ item.force_kill | default(true) }}"
+ # Modification: we want to clean up old volumes.
+ keep_volumes: false
+ register: server
+ with_items: "{{ molecule_yml.platforms }}"
+ async: 7200
+ poll: 0
+
+ - name: Wait for instance(s) deletion to complete
+ async_status:
+ jid: "{{ item.ansible_job_id }}"
+ register: docker_jobs
+ until: docker_jobs.finished
+ retries: 300
+ with_items: "{{ server.results }}"
+
+ - name: Delete docker network(s)
+ docker_network:
+ name: "{{ item }}"
+ docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+ state: absent
+ with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks }}"
diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml
new file mode 100644
index 00000000..e8e5ad76
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/molecule.yml
@@ -0,0 +1,78 @@
+---
+dependency:
+ name: galaxy
+driver:
+ name: docker
+lint:
+ name: yamllint
+platforms:
+ - name: infrastructure-server
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ env:
+ container: docker
+ groups:
+ - infrastructure
+ - kubernetes-control-plane
+ networks:
+ - name: rke
+ purge_networks: true
+
+ - name: kubernetes-node-1
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ env:
+ container: docker
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ groups:
+ - kubernetes
+ networks:
+ - name: rke
+ purge_networks: true
+
+ - name: kubernetes-node-2
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ env:
+ container: docker
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ groups:
+ - kubernetes
+ networks:
+ - name: rke
+ purge_networks: true
+
+provisioner:
+ name: ansible
+ env:
+ ANSIBLE_ROLES_PATH: ../../../../test/roles
+ ANSIBLE_LIBRARY: ../../../../library
+ inventory:
+ links:
+ group_vars: ../../../../group_vars
+ options:
+ e: "app_data_path=/opt/onap"
+ lint:
+ name: ansible-lint
+scenario:
+ name: default
+verifier:
+ name: testinfra
+ lint:
+ name: flake8
diff --git a/ansible/roles/rke/molecule/default/playbook.yml b/ansible/roles/rke/molecule/default/playbook.yml
new file mode 100644
index 00000000..09dbfb8e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/playbook.yml
@@ -0,0 +1,30 @@
+---
+- name: "Set cluster_ip"
+ hosts: all
+ tasks:
+ - name: "Set cluster_ip fact"
+ set_fact:
+ cluster_ip: "{{ ansible_default_ipv4.address }}"
+
+- name: Configure kubernetes cluster (RKE)
+ hosts: infrastructure
+ roles:
+ - role: rke
+ vars:
+ mode: config
+
+- name: Prepare kubernetes nodes (RKE)
+ hosts:
+ - kubernetes
+ - kubernetes-control-plane
+ roles:
+ - role: rke
+ vars:
+ mode: node
+
+- name: Deploy kubernetes cluster (RKE)
+ hosts: infrastructure
+ roles:
+ - role: rke
+ vars:
+ mode: deploy
diff --git a/ansible/roles/rke/molecule/default/prepare.yml b/ansible/roles/rke/molecule/default/prepare.yml
new file mode 100644
index 00000000..6bad2b80
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/prepare.yml
@@ -0,0 +1,15 @@
+---
+- name: "Prepare hosts"
+ hosts: all
+ roles:
+ - role: prepare-rke
+ vars:
+ mode: all
+ - prepare-docker-dind
+
+- name: "Infra specific preparations"
+ hosts: infrastructure
+ roles:
+ - role: prepare-rke
+ vars:
+ mode: infra
diff --git a/ansible/roles/rke/molecule/default/tests/test_controlplane.py b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
new file mode 100644
index 00000000..0bfbca2d
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
@@ -0,0 +1,14 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts(
+ 'kubernetes-control-plane')
+
+
+@pytest.mark.parametrize('container_name', [
+ 'kube-apiserver', 'kube-controller-manager', 'kube-scheduler', 'kubelet'])
+def test_container_running(host, container_name):
+ assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/default/tests/test_infrastructure.py b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
new file mode 100644
index 00000000..9ba11d6e
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
@@ -0,0 +1,56 @@
+import os
+import pytest
+import json
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('infrastructure')
+
+
+@pytest.mark.parametrize('filename', [
+ '/root/.kube/config',
+ '/opt/onap/cluster/cluster.yml',
+ '/opt/onap/cluster/cluster.rkestate'])
+def test_file_existence(host, filename):
+ assert host.file(filename).exists
+
+
+def test_rke_in_path(host):
+ assert host.find_command('rke') == '/usr/local/bin/rke'
+
+
+def test_rke_version_works(host):
+ # Note that we need to cd to the cluster data dir first, really.
+ assert host.run('cd /opt/onap/cluster && rke version').rc == 0
+
+
+def test_nodes_ready(host):
+ # Retrieve all node names.
+ nodecmdres = host.run('kubectl get nodes -o name')
+ assert nodecmdres.rc == 0
+ nodes = nodecmdres.stdout.split('\n')
+ for node in nodes:
+ assert host.run(
+ 'kubectl wait --timeout=0 --for=condition=ready ' + node).rc == 0
+
+
+def test_pods_ready(host):
+ # Retrieve all pods from all namespaces.
+ # Because we need pod and namespace name, we get full json representation.
+ podcmdres = host.run('kubectl get pods --all-namespaces -o json')
+ assert podcmdres.rc == 0
+ pods = json.loads(podcmdres.stdout)['items']
+ for pod in pods:
+ # Each pod may be either created by a job or not.
+ # In job case they should already be completed
+ # when we are here so we ignore them.
+ namespace = pod['metadata']['namespace']
+ podname = pod['metadata']['name']
+ condition = 'Ready'
+ if len(pod['metadata']['ownerReferences']) == 1 and pod[
+ 'metadata']['ownerReferences'][0]['kind'] == 'Job':
+ continue
+ assert host.run(
+ 'kubectl wait --timeout=120s --for=condition=' + condition + ' -n ' +
+ namespace + ' pods/' + podname).rc == 0
diff --git a/ansible/roles/rke/molecule/default/tests/test_kubernetes.py b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
new file mode 100644
index 00000000..887494fa
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
@@ -0,0 +1,13 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes')
+
+
+@pytest.mark.parametrize('container_name', [
+ 'etcd', 'kubelet', 'kube-proxy'])
+def test_container_running(host, container_name):
+ assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/tasks/rke_config.yml b/ansible/roles/rke/tasks/rke_config.yml
index 4112e107..9dc0d8c6 100644
--- a/ansible/roles/rke/tasks/rke_config.yml
+++ b/ansible/roles/rke/tasks/rke_config.yml
@@ -37,6 +37,7 @@
template:
src: cluster.yml.j2
dest: "{{ cluster_config_dir }}/cluster.yml"
+ register: cluster_yml
- name: Prepare rke addon manifest (dashboard)
template:
diff --git a/ansible/roles/rke/tasks/rke_deploy.yml b/ansible/roles/rke/tasks/rke_deploy.yml
index 9983d08a..7b36f55d 100644
--- a/ansible/roles/rke/tasks/rke_deploy.yml
+++ b/ansible/roles/rke/tasks/rke_deploy.yml
@@ -1,8 +1,17 @@
---
+- name: "Check if rke is deployed"
+ command: "rke version"
+ args:
+ chdir: "{{ cluster_config_dir }}"
+ failed_when: false
+ changed_when: false
+ register: rke_deployed
+
- name: Run rke up
command: "{{ rke_bin_dir }}/rke up --config cluster.yml"
args:
chdir: "{{ cluster_config_dir }}"
+ when: rke_deployed.rc != 0 or cluster_yml.changed # noqa 503
- name: Ensure .kube directory is present
file: