Update the Kubernetes Version to 1.16

Update kubespray to 2.12 to deploy Kubernetes 1.16

Issue-ID: MULTICLOUD-1063
Signed-off-by: Yao Le <le.yao@intel.com>
Change-Id: I537f6395e5d05d8b72411dd1e0789e19972f1947

16 files changed, 144 insertions, 21 deletions

diff --git a/deployments/helm/onap4k8s/charts/common/templates/_ingress.tpl b/deployments/helm/onap4k8s/charts/common/templates/_ingress.tpl
index 9b274ad1..aeae726e 100644
--- a/deployments/helm/onap4k8s/charts/common/templates/_ingress.tpl
+++ b/deployments/helm/onap4k8s/charts/common/templates/_ingress.tpl
@@ -29,7 +29,7 @@
 {{- define "common.ingress" -}}
 {{- if .Values.ingress -}}
 {{- if .Values.ingress.enabled -}}
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: {{ include "common.fullname" . }}-ingress
diff --git a/deployments/helm/onap4k8s/charts/etcd/Chart.yaml b/deployments/helm/onap4k8s/charts/etcd/Chart.yaml
index 7da825ac..012a4e13 100644
--- a/deployments/helm/onap4k8s/charts/etcd/Chart.yaml
+++ b/deployments/helm/onap4k8s/charts/etcd/Chart.yaml
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+apiVersion: v1
 name: etcd
 home: https://github.com/coreos/etcd
 version: 5.0.0
diff --git a/deployments/helm/onap4k8s/charts/etcd/templates/statefulset.yaml b/deployments/helm/onap4k8s/charts/etcd/templates/statefulset.yaml
index 7190c5bc..a32f0485 100644
--- a/deployments/helm/onap4k8s/charts/etcd/templates/statefulset.yaml
+++ b/deployments/helm/onap4k8s/charts/etcd/templates/statefulset.yaml
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: {{ include "common.fullname" .  }}
@@ -23,6 +23,12 @@ metadata:
 spec:
   serviceName: {{ include "common.servicename" .}}
   replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      heritage: "{{ .Release.Service }}"
+      release: "{{ .Release.Name }}"
+      chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+      app: {{ include "common.name" . }}
   template:
     metadata:
       labels:
diff --git a/deployments/helm/onap4k8s/charts/mongo/templates/nfs-provisoner.yaml b/deployments/helm/onap4k8s/charts/mongo/templates/nfs-provisoner.yaml
index 355ad382..a69382e4 100644
--- a/deployments/helm/onap4k8s/charts/mongo/templates/nfs-provisoner.yaml
+++ b/deployments/helm/onap4k8s/charts/mongo/templates/nfs-provisoner.yaml
@@ -15,7 +15,7 @@
 */}}
 
 {{ if not .Values.disableNfsProvisioner }}
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ .Values.nfsprovisionerPrefix }}-nfs-provisioner
@@ -26,6 +26,10 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.nfsprovisionerPrefix }}-nfs-provisioner
+      release: {{ .Release.Name }}
   #replicas: {{ .Values.replicaCount }}
   strategy:
     type: Recreate
diff --git a/deployments/helm/onap4k8s/charts/mongo/templates/statefulset.yaml b/deployments/helm/onap4k8s/charts/mongo/templates/statefulset.yaml
index c79739f5..c234ce3e 100644
--- a/deployments/helm/onap4k8s/charts/mongo/templates/statefulset.yaml
+++ b/deployments/helm/onap4k8s/charts/mongo/templates/statefulset.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 */}}
 
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: {{ include "common.fullname" . }}
@@ -27,6 +27,10 @@ metadata:
 spec:
   serviceName: {{ .Values.service.name }}
   replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "common.name" . }}
+      release: {{ .Release.Name }}
   template:
     metadata:
       labels:
diff --git a/deployments/helm/onap4k8s/charts/multicloud-k8s/templates/deployment.yaml b/deployments/helm/onap4k8s/charts/multicloud-k8s/templates/deployment.yaml
index 846a7513..907afd32 100644
--- a/deployments/helm/onap4k8s/charts/multicloud-k8s/templates/deployment.yaml
+++ b/deployments/helm/onap4k8s/charts/multicloud-k8s/templates/deployment.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 */}}
 
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "common.fullname" . }}
@@ -26,6 +26,10 @@ metadata:
     heritage: {{ .Release.Service }}
 spec:
   replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "common.name" . }}
+      release: {{ .Release.Name }}
   template:
     metadata:
       labels:
diff --git a/kud/ci/k8s-cluster.yml b/kud/ci/k8s-cluster.yml
new file mode 100644
index 00000000..8191d4b4
--- /dev/null
+++ b/kud/ci/k8s-cluster.yml
@@ -0,0 +1,83 @@
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Kubernetes configuration dirs and system namespace.
+# Those are where all the additional config stuff goes
+# kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location and namespace.
+# Editing those values will almost surely break something.
+system_namespace: kube-system
+
+docker_version: 'latest'
+
+# Logging directory (sysvinit systems)
+kube_log_dir: "/var/log/kubernetes"
+
+kube_api_anonymous_auth: true
+
+# Users to create for basic auth in Kubernetes API via HTTP
+# Optionally add groups for user
+kube_api_pwd: "secret"
+kube_users:
+  kube:
+    pass: "{{kube_api_pwd}}"
+    role: admin
+    groups:
+      - system:masters
+
+## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth)
+#kube_oidc_auth: false
+kube_basic_auth: true
+kube_token_auth: true
+
+# Choose network plugin (calico, contiv, weave or flannel)
+# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
+kube_network_plugin: flannel
+
+# Make a copy of kubeconfig (admin.conf) on the host that runs Ansible to inventory/artifacts
+kubeconfig_localhost: true
+# Copy kubectl binary on the host that runs Ansible to inventory/artifacts
+kubectl_localhost: true
+# Disable nodelocal dns cache
+enable_nodelocaldns: false
+# Enable MountPropagation gate feature
+local_volumes_enabled: true
+local_volume_provisioner_enabled: true
+
+## Change this to use another Kubernetes version, e.g. a current beta release
+kube_version: v1.16.9
+
+# Helm deployment
+helm_enabled: true
+
+# Kube-proxy proxyMode configuration.
+# NOTE: Ipvs is based on netfilter hook function, but uses hash table as the underlying data structure and
+# works in the kernel space
+# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs
+#kube_proxy_mode: ipvs
+
+# Download container images only once then push to cluster nodes in batches
+download_run_once: False
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+download_cache_dir: "/tmp/kubespray_cache"
+retry_stagger: 10
+
+#Set download_localhost: True to make localhost the download delegate. This can be useful if 
+#cluster nodes cannot access external addresses. To use this requires that docker is installed 
+#and running on the ansible master and that the current user is either in the docker group or 
+#can do passwordless sudo, to be able to access docker. 
+download_localhost: False
+
+# Subnet for cluster IPs
+kube_service_addresses: 10.244.0.0/18
+# Subnet for Pod IPs
+kube_pods_subnet: 10.244.64.0/18
diff --git a/kud/ci/kud-installer.sh b/kud/ci/kud-installer.sh
index 73c1628b..956a15a6 100755
--- a/kud/ci/kud-installer.sh
+++ b/kud/ci/kud-installer.sh
@@ -9,5 +9,6 @@
 set -x -e -o pipefail
 
 curr_dir="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
+cp ${curr_dir}/k8s-cluster.yml ${curr_dir}/../hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
 cd ${curr_dir}/../hosting_providers/baremetal
 ./aio.sh
diff --git a/kud/deployment_infra/images/multus-daemonset.yml b/kud/deployment_infra/images/multus-daemonset.yml
index 0c41a052..d6d8d533 100644
--- a/kud/deployment_infra/images/multus-daemonset.yml
+++ b/kud/deployment_infra/images/multus-daemonset.yml
@@ -101,7 +101,7 @@ data:
       "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
     }
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-multus-ds-amd64
@@ -110,6 +110,10 @@ metadata:
     tier: node
     app: multus
 spec:
+  selector:
+    matchLabels:
+      tier: node
+      app: multus
   updateStrategy:
     type: RollingUpdate
   template:
diff --git a/kud/deployment_infra/images/nfn.yml b/kud/deployment_infra/images/nfn.yml
index 2e033e04..5c5e4442 100644
--- a/kud/deployment_infra/images/nfn.yml
+++ b/kud/deployment_infra/images/nfn.yml
@@ -448,7 +448,7 @@ data:
           kubeconfig=/etc/kubernetes/admin.conf
 
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: ovn4nfv-cni
@@ -456,6 +456,9 @@ metadata:
   labels:
     app: ovn4nfv
 spec:
+  selector:
+    matchLabels:
+      app: ovn4nfv
   updateStrategy:
     type: RollingUpdate
   template:
@@ -505,7 +508,7 @@ spec:
               path: ovn4nfv_k8s.conf
 
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: nfn-agent
@@ -513,6 +516,9 @@ metadata:
   labels:
     app: nfn-agent
 spec:
+  selector:
+    matchLabels:
+      app: nfn-agent
   updateStrategy:
     type: RollingUpdate
   template:
diff --git a/kud/deployment_infra/images/sriov-cni.yml b/kud/deployment_infra/images/sriov-cni.yml
index bd943d04..7503b872 100644
--- a/kud/deployment_infra/images/sriov-cni.yml
+++ b/kud/deployment_infra/images/sriov-cni.yml
@@ -2,7 +2,7 @@
 # Based on:
 # https://github.com/intel/sriov-cni/blob/master/images/sriov-cni-daemonset.yaml
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-sriov-cni-ds-amd64
@@ -11,6 +11,10 @@ metadata:
     tier: node
     app: sriov-cni
 spec:
+  selector:
+    matchLabels:
+      tier: node
+      app: sriov-cni
   template:
     metadata:
       labels:
diff --git a/kud/deployment_infra/images/sriov-daemonset.yml b/kud/deployment_infra/images/sriov-daemonset.yml
index 72f33869..e392028d 100644
--- a/kud/deployment_infra/images/sriov-daemonset.yml
+++ b/kud/deployment_infra/images/sriov-daemonset.yml
@@ -27,7 +27,7 @@ metadata:
   namespace: kube-system
 
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-sriov-device-plugin-amd64
@@ -36,6 +36,10 @@ metadata:
     tier: node
     app: sriovdp
 spec:
+  selector:
+    matchLabels:
+      tier: node
+      app: sriovdp
   template:
     metadata:
       labels:
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 7a25670d..89c8f4ea 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -57,10 +57,10 @@ cmk_untaint_required: true
 #cmk_exclusive_mode: packed # choose between: packed, spread, default: packed
 
 go_version: '1.12.5'
-kubespray_version: 2.10.4
+kubespray_version: 2.12.6
 helm_client_version: 2.13.1
 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634
-ansible_version: 2.7.10
+ansible_version: 2.9.7
 
 sriov_dest: "{{ base_dest }}/sriov"
 sriov_driver_source_type: "tarball"
diff --git a/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
index 5ba115b6..5560dd97 100644
--- a/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
+++ b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
@@ -14,6 +14,8 @@
 # Editing those values will almost surely break something.
 system_namespace: kube-system
 
+docker_version: 'latest'
+
 # Logging directory (sysvinit systems)
 kube_log_dir: "/var/log/kubernetes"
 
@@ -48,7 +50,7 @@ local_volumes_enabled: true
 local_volume_provisioner_enabled: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.14.3
+kube_version: v1.16.9
 
 # Helm deployment
 helm_enabled: true
diff --git a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
index 18339773..30fd5c0b 100644
--- a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
+++ b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
@@ -51,7 +51,7 @@ local_volumes_enabled: true
 local_volume_provisioner_enabled: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.14.3
+kube_version: v1.16.9
 
 # Helm deployment
 helm_enabled: true
@@ -63,17 +63,17 @@ helm_enabled: true
 #kube_proxy_mode: ipvs
 
 # Download container images only once then push to cluster nodes in batches
-download_run_once: true
+download_run_once: True
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"
 
-# Makes the installer node a delegate for pushing images while running
-# the deployment with ansible. This maybe the case if cluster nodes
-# cannot access each over via ssh or you want to use local docker
-# images as a cache for multiple clusters.
-download_localhost: true
+#Set download_localhost: True to make localhost the download delegate. This can be useful if 
+#cluster nodes cannot access external addresses. To use this requires that docker is installed 
+#and running on the ansible master and that the current user is either in the docker group or 
+#can do passwordless sudo, to be able to access docker. 
+download_localhost: True
 
 # Subnet for cluster IPs
 kube_service_addresses: 10.244.0.0/18
diff --git a/kud/tests/vnfs/comp-app/collection/app1/helm/collectd/templates/daemonset.yaml b/kud/tests/vnfs/comp-app/collection/app1/helm/collectd/templates/daemonset.yaml
index bc686381..6935398b 100644
--- a/kud/tests/vnfs/comp-app/collection/app1/helm/collectd/templates/daemonset.yaml
+++ b/kud/tests/vnfs/comp-app/collection/app1/helm/collectd/templates/daemonset.yaml
@@ -14,7 +14,7 @@
 # limitations under the License.
 */}}
 
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: {{ template "fullname" . }}