From 81a5673727746e4975ff77bde3503ac0319a95a4 Mon Sep 17 00:00:00 2001 From: Michal Zegan Date: Tue, 28 May 2019 17:05:01 +0200 Subject: Add kubernetes-node group for workers Moved all kubernetes nodes from the kubernetes to kubernetes-node group, and made the kubernetes group a parent of both kubernetes-node and kubernetes-control-plane. The reason is that we still need to have separate groups for control planes and for nodes, but some operations are performed equally on any kind of kubernetes cluster member, and currently one would need to separately include all related groups one by one. Example of such common operation is installation of docker, that was not performed for kubernetes-control-plane group, so that docker was not installed on control planes that were neither infra nor nodes. The side effect of this change is that if infrastructure server is also a cluster member, it lands in a kubernetes group by default, and node specific setup would be performed on it. For that reason, playbook is modified to perform this setup on all kubernetes cluster members except the infra server. Change-Id: Ic827002d28e535334dbab7e5ad4aed8aa95c97a9 Issue-ID: OOM-1778 Signed-off-by: Michal Zegan --- ansible/infrastructure.yml | 2 +- ansible/inventory/hosts.yml | 29 ++++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/ansible/infrastructure.yml b/ansible/infrastructure.yml index ce4d4d72..7fdbd2e1 100644 --- a/ansible/infrastructure.yml +++ b/ansible/infrastructure.yml @@ -18,7 +18,7 @@ - nexus - name: Setup base for Kubernetes nodes - hosts: kubernetes + hosts: kubernetes:!infrastructure roles: - chrony - package-repository-check diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index 37ae4e39..11825500 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -25,20 +25,23 @@ all: # This is group of hosts which are/will be part of Kubernetes cluster. kubernetes: - hosts: - kubernetes-node-1: - ansible_host: 10.8.8.19 - #ip of the node that it uses for communication with k8s cluster. - cluster_ip: 10.8.8.19 + children: + # This is a group of hosts containing kubernetes worker nodes. + kubernetes-node: + hosts: + kubernetes-node-1: + ansible_host: 10.8.8.19 + #ip of the node that it uses for communication with k8s cluster. + cluster_ip: 10.8.8.19 - # This is a group of hosts that are to be used as kubernetes control plane nodes. - # This means they host kubernetes api server, controller manager and scheduler. - # This example uses infra for this purpose, however note that any - # other host could be used including kubernetes nodes. - # cluster_ip needs to be set for hosts used as control planes. - kubernetes-control-plane: - hosts: - infrastructure-server + # This is a group of hosts that are to be used as kubernetes control plane nodes. + # This means they host kubernetes api server, controller manager and scheduler. + # This example uses infra for this purpose, however note that any + # other host could be used including kubernetes nodes. + # cluster_ip needs to be set for hosts used as control planes. + kubernetes-control-plane: + hosts: + infrastructure-server nfs-server: hosts: -- cgit 1.2.3-korg From 89e6314a29895247013929828817acbc4a1f6ea5 Mon Sep 17 00:00:00 2001 From: Michal Zegan Date: Tue, 28 May 2019 17:16:47 +0200 Subject: Make rke to use new group structure Rke playbook/role is modified to take advantage of the new group structure. Namely, all members of kubernetes group are treated as cluster members independent of their role. The role itself is selected in cluster.yml.j2 template based on actual membership in either kubernetes-node or kubernetes-control-plane group. Change-Id: I9a5bbfd090aff17018a610a01d6f88d848fc26c4 Issue-ID: OOM-1778 Signed-off-by: Michal Zegan --- ansible/rke.yml | 6 ++---- ansible/roles/rke/molecule/default/molecule.yml | 3 +++ ansible/roles/rke/molecule/default/playbook.yml | 6 ++---- ansible/roles/rke/molecule/default/tests/test_kubernetes.py | 13 ------------- ansible/roles/rke/molecule/default/tests/test_nodes.py | 13 +++++++++++++ ansible/roles/rke/templates/cluster.yml.j2 | 6 ++---- 6 files changed, 22 insertions(+), 25 deletions(-) delete mode 100644 ansible/roles/rke/molecule/default/tests/test_kubernetes.py create mode 100644 ansible/roles/rke/molecule/default/tests/test_nodes.py diff --git a/ansible/rke.yml b/ansible/rke.yml index e0d6dcf1..13e7bb5b 100644 --- a/ansible/rke.yml +++ b/ansible/rke.yml @@ -9,10 +9,8 @@ vars: mode: config -- name: Prepare kubernetes nodes (RKE) - hosts: - - kubernetes - - kubernetes-control-plane +- name: Prepare kubernetes hosts (RKE) + hosts: kubernetes roles: - role: rke vars: diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml index e8e5ad76..444a7519 100644 --- a/ansible/roles/rke/molecule/default/molecule.yml +++ b/ansible/roles/rke/molecule/default/molecule.yml @@ -20,6 +20,7 @@ platforms: groups: - infrastructure - kubernetes-control-plane + - kubernetes networks: - name: rke purge_networks: true @@ -37,6 +38,7 @@ platforms: - /var/lib/docker groups: - kubernetes + - kubernetes-node networks: - name: rke purge_networks: true @@ -54,6 +56,7 @@ platforms: - /var/lib/docker groups: - kubernetes + - kubernetes-node networks: - name: rke purge_networks: true diff --git a/ansible/roles/rke/molecule/default/playbook.yml b/ansible/roles/rke/molecule/default/playbook.yml index 09dbfb8e..fab7a0d0 100644 --- a/ansible/roles/rke/molecule/default/playbook.yml +++ b/ansible/roles/rke/molecule/default/playbook.yml @@ -13,10 +13,8 @@ vars: mode: config -- name: Prepare kubernetes nodes (RKE) - hosts: - - kubernetes - - kubernetes-control-plane +- name: Prepare kubernetes hosts (RKE) + hosts: kubernetes roles: - role: rke vars: diff --git a/ansible/roles/rke/molecule/default/tests/test_kubernetes.py b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py deleted file mode 100644 index 887494fa..00000000 --- a/ansible/roles/rke/molecule/default/tests/test_kubernetes.py +++ /dev/null @@ -1,13 +0,0 @@ -import os -import pytest - -import testinfra.utils.ansible_runner - -testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( - os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes') - - -@pytest.mark.parametrize('container_name', [ - 'etcd', 'kubelet', 'kube-proxy']) -def test_container_running(host, container_name): - assert host.docker(container_name).is_running diff --git a/ansible/roles/rke/molecule/default/tests/test_nodes.py b/ansible/roles/rke/molecule/default/tests/test_nodes.py new file mode 100644 index 00000000..fcc5e7e6 --- /dev/null +++ b/ansible/roles/rke/molecule/default/tests/test_nodes.py @@ -0,0 +1,13 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes-node') + + +@pytest.mark.parametrize('container_name', [ + 'etcd', 'kubelet', 'kube-proxy']) +def test_container_running(host, container_name): + assert host.docker(container_name).is_running diff --git a/ansible/roles/rke/templates/cluster.yml.j2 b/ansible/roles/rke/templates/cluster.yml.j2 index 64508e6f..f28cff80 100644 --- a/ansible/roles/rke/templates/cluster.yml.j2 +++ b/ansible/roles/rke/templates/cluster.yml.j2 @@ -1,7 +1,5 @@ nodes: -{# Note that we iterate through all nodes in relevant groups. -We check which groups they belong to exactly later to determine roles. #} -{% for node in groups['kubernetes'] | union(groups['kubernetes-control-plane']) %} +{% for node in groups['kubernetes'] %} - address: "{{ hostvars[node].cluster_ip }}" port: "22" internal_address: "{{ hostvars[node].cluster_ip }}" @@ -9,7 +7,7 @@ We check which groups they belong to exactly later to determine roles. #} {% if node in groups['kubernetes-control-plane'] %} - controlplane {% endif %} -{% if node in groups['kubernetes'] %} +{% if node in groups['kubernetes-node'] %} - worker - etcd {% endif %} -- cgit 1.2.3-korg From bfbb6e193f28e7e503e4c94e46017d40fd613872 Mon Sep 17 00:00:00 2001 From: Michal Zegan Date: Mon, 3 Jun 2019 14:54:02 +0200 Subject: Add kubernetes-etcd group This group will contain kubernetes etcd cluster. It is separated from kubernetes control plane and, especially, nodes. There are of course no restrictions as to which machines can be etcd. Default is infrastructure-server. Change-Id: I8d3ab9b9e4680f57ea8f595d7be3ed6e2d32764c Issue-ID: OOM-1778 Signed-off-by: Michal Zegan --- ansible/inventory/hosts.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/inventory/hosts.yml b/ansible/inventory/hosts.yml index 11825500..4a6b68cf 100644 --- a/ansible/inventory/hosts.yml +++ b/ansible/inventory/hosts.yml @@ -34,6 +34,12 @@ all: #ip of the node that it uses for communication with k8s cluster. cluster_ip: 10.8.8.19 + # Group of hosts containing etcd cluster nodes. + # Defaults to infra. + kubernetes-etcd: + hosts: + infrastructure-server + # This is a group of hosts that are to be used as kubernetes control plane nodes. # This means they host kubernetes api server, controller manager and scheduler. # This example uses infra for this purpose, however note that any -- cgit 1.2.3-korg From 2f97acdcc45c70ee2fa022fdeb27b68a0f7e458b Mon Sep 17 00:00:00 2001 From: Michal Zegan Date: Mon, 3 Jun 2019 14:58:20 +0200 Subject: Add etcd group support to rke role Etcd cluster is now placed in members of kubernetes-etcd group instead of kubernetes-node. Due to defaults, it means infra is the only etcd by default. Change-Id: Iae05a42442849e4a248d73c2d78f5e2b0eae7255 Issue-ID: OOM-1778 Signed-off-by: Michal Zegan --- ansible/roles/rke/molecule/default/molecule.yml | 1 + ansible/roles/rke/molecule/default/tests/test_etcd.py | 13 +++++++++++++ ansible/roles/rke/molecule/default/tests/test_nodes.py | 2 +- ansible/roles/rke/templates/cluster.yml.j2 | 2 ++ 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/rke/molecule/default/tests/test_etcd.py diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml index 444a7519..6ae613a9 100644 --- a/ansible/roles/rke/molecule/default/molecule.yml +++ b/ansible/roles/rke/molecule/default/molecule.yml @@ -19,6 +19,7 @@ platforms: container: docker groups: - infrastructure + - kubernetes-etcd - kubernetes-control-plane - kubernetes networks: diff --git a/ansible/roles/rke/molecule/default/tests/test_etcd.py b/ansible/roles/rke/molecule/default/tests/test_etcd.py new file mode 100644 index 00000000..0f4b6f12 --- /dev/null +++ b/ansible/roles/rke/molecule/default/tests/test_etcd.py @@ -0,0 +1,13 @@ +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes-etcd') + + +@pytest.mark.parametrize('container_name', [ + 'etcd']) +def test_container_running(host, container_name): + assert host.docker(container_name).is_running diff --git a/ansible/roles/rke/molecule/default/tests/test_nodes.py b/ansible/roles/rke/molecule/default/tests/test_nodes.py index fcc5e7e6..60413018 100644 --- a/ansible/roles/rke/molecule/default/tests/test_nodes.py +++ b/ansible/roles/rke/molecule/default/tests/test_nodes.py @@ -8,6 +8,6 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( @pytest.mark.parametrize('container_name', [ - 'etcd', 'kubelet', 'kube-proxy']) + 'kubelet', 'kube-proxy']) def test_container_running(host, container_name): assert host.docker(container_name).is_running diff --git a/ansible/roles/rke/templates/cluster.yml.j2 b/ansible/roles/rke/templates/cluster.yml.j2 index f28cff80..2012ab92 100644 --- a/ansible/roles/rke/templates/cluster.yml.j2 +++ b/ansible/roles/rke/templates/cluster.yml.j2 @@ -9,6 +9,8 @@ nodes: {% endif %} {% if node in groups['kubernetes-node'] %} - worker +{% endif %} +{% if node in groups['kubernetes-etcd'] %} - etcd {% endif %} hostname_override: "" -- cgit 1.2.3-korg