Upgrade kubespray from 2.12.6 to 2.14.1

- Replace move of ansible.cfg from kubespray distribution to
  /etc/ansible with ANSIBLE_CONFIG environment variable.  Ansible
  modifies ansible.cfg during installation, and the paths in it are
  relative.

- kubespray 2.14.1 requires a kubernetes version > 1.16.  Use the
  default versions of kubernetes and helm provided by kubespray
  2.14.1.

- kubespray 2.14.1 replaces helm 2 with helm 3.  This removes support
  for helm init and helm serve.  It is no longer necessary to call
  helm init, and the helm serve repository is replaced with file
  relative URLs.  This also triggered a subsequent update of the
  kubernetes-helm ansible module to include the newer helm versions.

- Add "storageType: hostPath" to etcd/values.yaml.  Helm deploy of
  etcd will fail without this due to nil
  PersistentVolume.metadata.labels.type.

- The mitogen module used by kubespray/ansible requires python2 on the
  hosts.  Use the linear strategy to bypass mitogen and install
  python2 on the cluster hosts.

Issue-ID: MULTICLOUD-1230
Signed-off-by: Todd Malsbary <todd.malsbary@intel.com>
Change-Id: I9f50bb4e123fdcacab6b6a97e79cd09fb5c96634

33 files changed, 75 insertions, 123 deletions

diff --git a/deployments/helm/onap4k8s/Makefile b/deployments/helm/onap4k8s/Makefile
index 20470878..f078765d 100644
--- a/deployments/helm/onap4k8s/Makefile
+++ b/deployments/helm/onap4k8s/Makefile
@@ -54,16 +54,5 @@ clean:
 	@rm -rf $(PACKAGE_DIR)/*
 	@rm -rf $(OUTPUT_DIR)
 
-# start up a local helm repo to serve up helm chart packages
-repo:
-	@mkdir -p $(PACKAGE_DIR)
-	@helm serve --repo-path $(PACKAGE_DIR) &
-	@helm repo index $(PACKAGE_DIR)
-	@helm repo add onap4k8s http://127.0.0.1:8879
-
-# stop local helm repo
-repo-stop:
-	@pkill helm
-	@helm repo remove onap4k8s
 %:
 	@:
diff --git a/deployments/helm/onap4k8s/README.txt b/deployments/helm/onap4k8s/README.txt
index 857b0a0b..1528c740 100644
--- a/deployments/helm/onap4k8s/README.txt
+++ b/deployments/helm/onap4k8s/README.txt
@@ -16,9 +16,6 @@
 # Installation of ONAP4K8S helm chart
 #################################################################
 
-1. Create a helm repo (onap4k8s) from Makefile
-$ make repo
-
 1. Run "Makefile" in ONAP4K8S repo
 $ make all
 
diff --git a/deployments/helm/onap4k8s/charts/etcd/requirements.yaml b/deployments/helm/onap4k8s/charts/etcd/requirements.yaml
index ff2221f6..8c46d632 100644
--- a/deployments/helm/onap4k8s/charts/etcd/requirements.yaml
+++ b/deployments/helm/onap4k8s/charts/etcd/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~5.x-0
-    repository: '@onap4k8s'
+    repository: 'file://../common'
diff --git a/deployments/helm/onap4k8s/charts/etcd/values.yaml b/deployments/helm/onap4k8s/charts/etcd/values.yaml
index 7f53d222..ab891da7 100644
--- a/deployments/helm/onap4k8s/charts/etcd/values.yaml
+++ b/deployments/helm/onap4k8s/charts/etcd/values.yaml
@@ -55,6 +55,7 @@ persistence:
   ##
   #storageClass: "-"
   accessMode: "ReadWriteOnce"
+  storageType: hostPath
   storage: "1Gi"
   mountPath: /dockerdata-nfs
   mountSubPath: k8s-etcd
diff --git a/deployments/helm/onap4k8s/charts/mongo/requirements.yaml b/deployments/helm/onap4k8s/charts/mongo/requirements.yaml
index 0693dd3f..ed887924 100644
--- a/deployments/helm/onap4k8s/charts/mongo/requirements.yaml
+++ b/deployments/helm/onap4k8s/charts/mongo/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~5.x-0
-    repository: '@onap4k8s'
+    repository: 'file://../common'
diff --git a/deployments/helm/onap4k8s/charts/multicloud-k8s/requirements.yaml b/deployments/helm/onap4k8s/charts/multicloud-k8s/requirements.yaml
index c5102e49..b114c96e 100644
--- a/deployments/helm/onap4k8s/charts/multicloud-k8s/requirements.yaml
+++ b/deployments/helm/onap4k8s/charts/multicloud-k8s/requirements.yaml
@@ -18,10 +18,10 @@ dependencies:
     # local reference to common chart, as it is
     # a part of this chart's package and will not
     # be published independently to a repo (at this point)
-    repository: '@onap4k8s'
+    repository: 'file://../common'
   - name: mongo
     version: ~5.x-0
-    repository: '@onap4k8s'
+    repository: 'file://../mongo'
   - name: etcd
     version: ~5.x-0
-    repository: '@onap4k8s'
+    repository: 'file://../etcd'
diff --git a/deployments/helm/v2/emco/Makefile b/deployments/helm/v2/emco/Makefile
index dc88ff86..355fc3f6 100644
--- a/deployments/helm/v2/emco/Makefile
+++ b/deployments/helm/v2/emco/Makefile
@@ -64,16 +64,5 @@ clean:
 	@rm -rf $(PACKAGE_DIR)/*
 	@rm -rf $(OUTPUT_DIR)
 
-# start up a local helm repo to serve up helm chart packages
-repo:
-	@mkdir -p $(PACKAGE_DIR)
-	@helm serve --repo-path $(PACKAGE_DIR) &
-	@helm repo index $(PACKAGE_DIR)
-	@helm repo add local http://127.0.0.1:8879
-
-# stop local helm repo
-repo-stop:
-	@pkill helm
-	@helm repo remove local
 %:
 	@:
diff --git a/deployments/helm/v2/emco/README.md b/deployments/helm/v2/emco/README.md
index 407f4509..70cbfcde 100644
--- a/deployments/helm/v2/emco/README.md
+++ b/deployments/helm/v2/emco/README.md
@@ -6,11 +6,7 @@ EMCO Helm charts include charts for EMCO microservices along with MongoDb, etcd,
 
 
 ### Steps to generate and install packages
-**1. Create a local helm repo from Makefile**
-
-`$ make repo`
-
-**2. Run make file to package all the required chart**
+**1. Run make file to package all the required chart**
 
 `$ make clean`
 
@@ -26,13 +22,13 @@ Pacakges helm charts in tar.gz format. All packages are in **dist/packages** dir
   | **emco-0.1.0.tgz**   | Includes all charts including database, all services and tools        |
 
 
-**3. Deploy EMCO Packages for Databases and Services**
+**2. Deploy EMCO Packages for Databases and Services**
 
 `$ helm install dist/packages/emco-db-0.1.0.tgz --name emco-db --namespace emco`
 
 `$ helm install dist/packages/emco-services-0.1.0.tgz --name emco-services --namespace emco`
 
-**4. Deploy tools (Optional)**
+**3. Deploy tools (Optional)**
 
 `$ helm install dist/packages/emco-tools-0.1.0.tgz --name emco-tools --namespace emco`
 
@@ -41,12 +37,12 @@ NOTE: Deploy the Chart emco-0.1.0.tgz to deploy all packages including database,
 `$ helm install dist/packages/emco-0.1.0.tgz --name emco --namespace emco`
 
 
-**5. To check logs of the different Microservices check fluentd logs**
+**4. To check logs of the different Microservices check fluentd logs**
 
 `kubectl logs emco-tools-fluentd-0 -n emco | grep orchestrator`
 
 
-**6. Delete all packages**
+**5. Delete all packages**
 
 `$ helm delete emco-services --purge`
 
@@ -61,7 +57,7 @@ NOTE: If the Chart emco-0.1.0.tgz was deployed
 `$ helm delete emco --purge`
 
 
-**7. Delete local helm repo**
+**6. Delete local helm repo**
 
 `make repo-stop`
 
@@ -74,4 +70,4 @@ After deleting the db package and before installing the package again following
 Workarounds:
 
 * remove the  finalizers section using `kubectl edit persistentvolumes emco-db-emco-etcd-data-0`
-* or, if appropriate, delete the entire namespace using `kubectl delete namespace emco`
\ No newline at end of file
+* or, if appropriate, delete the entire namespace using `kubectl delete namespace emco`
diff --git a/deployments/helm/v2/emco/clm/requirements.yaml b/deployments/helm/v2/emco/clm/requirements.yaml
index c2253cf4..bba5c27d 100644
--- a/deployments/helm/v2/emco/clm/requirements.yaml
+++ b/deployments/helm/v2/emco/clm/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/dcm/requirements.yaml b/deployments/helm/v2/emco/dcm/requirements.yaml
index a3792606..c5c20127 100644
--- a/deployments/helm/v2/emco/dcm/requirements.yaml
+++ b/deployments/helm/v2/emco/dcm/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/emco-db/requirements.yaml b/deployments/helm/v2/emco/emco-db/requirements.yaml
index 52a40b64..62e807b6 100644
--- a/deployments/helm/v2/emco/emco-db/requirements.yaml
+++ b/deployments/helm/v2/emco/emco-db/requirements.yaml
@@ -15,10 +15,10 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
   - name: mongo
     version: ~4.x-0
-    repository: '@local'
+    repository: 'file://../mongo'
   - name: etcd
     version: ~3.x-0
-    repository: '@local'
+    repository: 'file://../etcd'
diff --git a/deployments/helm/v2/emco/emco-services/requirements.yaml b/deployments/helm/v2/emco/emco-services/requirements.yaml
index dbb14661..ddb9fb7f 100644
--- a/deployments/helm/v2/emco/emco-services/requirements.yaml
+++ b/deployments/helm/v2/emco/emco-services/requirements.yaml
@@ -15,22 +15,22 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
   - name: orchestrator
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../orchestrator'
   - name: ncm
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../ncm'
   - name: rsync
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../rsync'
   - name: clm
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../clm'
   - name: ovnaction
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../ovnaction'
   - name: dcm
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../dcm'
diff --git a/deployments/helm/v2/emco/emco-tools/requirements.yaml b/deployments/helm/v2/emco/emco-tools/requirements.yaml
index ddb91c51..14ad537d 100644
--- a/deployments/helm/v2/emco/emco-tools/requirements.yaml
+++ b/deployments/helm/v2/emco/emco-tools/requirements.yaml
@@ -15,7 +15,7 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
   - name: fluentd
     version: ~1.x-0
-    repository: '@local'
+    repository: 'file://../fluentd'
diff --git a/deployments/helm/v2/emco/emco/requirements.yaml b/deployments/helm/v2/emco/emco/requirements.yaml
index 21649c38..c704d2f2 100644
--- a/deployments/helm/v2/emco/emco/requirements.yaml
+++ b/deployments/helm/v2/emco/emco/requirements.yaml
@@ -15,11 +15,11 @@
 dependencies:
   - name: emco-db
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../emco-db'
   - name: emco-tools
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../emco-tools'
   - name: emco-services
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../emco-services'
 
diff --git a/deployments/helm/v2/emco/etcd/requirements.yaml b/deployments/helm/v2/emco/etcd/requirements.yaml
index d71b975c..9802dac4 100644
--- a/deployments/helm/v2/emco/etcd/requirements.yaml
+++ b/deployments/helm/v2/emco/etcd/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/etcd/values.yaml b/deployments/helm/v2/emco/etcd/values.yaml
index 226b4594..ef9494b9 100644
--- a/deployments/helm/v2/emco/etcd/values.yaml
+++ b/deployments/helm/v2/emco/etcd/values.yaml
@@ -48,6 +48,7 @@ persistence:
   enabled: false
   volumeReclaimPolicy: Retain
   accessMode:  ReadWriteMany
+  storageType: hostPath
   storage: 1Gi
   mountPath: /dockerdata-nfs
   mountSubPath: "etcd/data"
diff --git a/deployments/helm/v2/emco/mongo/requirements.yaml b/deployments/helm/v2/emco/mongo/requirements.yaml
index 38a832b8..b8b139fa 100644
--- a/deployments/helm/v2/emco/mongo/requirements.yaml
+++ b/deployments/helm/v2/emco/mongo/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/ncm/requirements.yaml b/deployments/helm/v2/emco/ncm/requirements.yaml
index c2253cf4..bba5c27d 100644
--- a/deployments/helm/v2/emco/ncm/requirements.yaml
+++ b/deployments/helm/v2/emco/ncm/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/orchestrator/requirements.yaml b/deployments/helm/v2/emco/orchestrator/requirements.yaml
index c2253cf4..bba5c27d 100644
--- a/deployments/helm/v2/emco/orchestrator/requirements.yaml
+++ b/deployments/helm/v2/emco/orchestrator/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/ovnaction/requirements.yaml b/deployments/helm/v2/emco/ovnaction/requirements.yaml
index c2253cf4..bba5c27d 100644
--- a/deployments/helm/v2/emco/ovnaction/requirements.yaml
+++ b/deployments/helm/v2/emco/ovnaction/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/deployments/helm/v2/emco/rsync/requirements.yaml b/deployments/helm/v2/emco/rsync/requirements.yaml
index c2253cf4..bba5c27d 100644
--- a/deployments/helm/v2/emco/rsync/requirements.yaml
+++ b/deployments/helm/v2/emco/rsync/requirements.yaml
@@ -15,4 +15,4 @@
 dependencies:
   - name: common
     version: ~0.x-0
-    repository: '@local'
+    repository: 'file://../common'
diff --git a/kud/deployment_infra/galaxy-requirements.yml b/kud/deployment_infra/galaxy-requirements.yml
index 3191dc19..9747dc99 100644
--- a/kud/deployment_infra/galaxy-requirements.yml
+++ b/kud/deployment_infra/galaxy-requirements.yml
@@ -10,6 +10,6 @@
 - src: andrewrothstein.go
   version: v2.1.15
 - src: andrewrothstein.kubernetes-helm
-  version: v1.2.17
+  version: v1.3.16
 - src: geerlingguy.docker
   version: 2.5.2
diff --git a/kud/deployment_infra/playbooks/configure-emco-reset.yml b/kud/deployment_infra/playbooks/configure-emco-reset.yml
index a4560cc2..7cad36e4 100644
--- a/kud/deployment_infra/playbooks/configure-emco-reset.yml
+++ b/kud/deployment_infra/playbooks/configure-emco-reset.yml
@@ -15,7 +15,7 @@
         file: kud-vars.yml
 
     - name: Change the emco directory and run helm delete
-      command: /usr/local/bin/helm delete --purge emco
+      command: /usr/local/bin/helm uninstall --namespace emco emco
       register: helm_delete
       args:
         chdir: /opt/multicloud/deployments/helm/v2/emco
@@ -41,15 +41,6 @@
     - debug:
         var: make_clean.stdout_lines
 
-    - name: Change the emco directory and make repo-stop
-      command: /usr/bin/make repo-stop
-      register: make_repo_stop
-      args:
-        chdir: /opt/multicloud/deployments/helm/v2/emco
-
-    - debug:
-        var: make_repo_stop.stdout_lines
-
     - name: clean multicloud-k8s path
       file:
         state: absent
diff --git a/kud/deployment_infra/playbooks/configure-emco.yml b/kud/deployment_infra/playbooks/configure-emco.yml
index 255ce6a4..eb6adcf2 100644
--- a/kud/deployment_infra/playbooks/configure-emco.yml
+++ b/kud/deployment_infra/playbooks/configure-emco.yml
@@ -27,15 +27,6 @@
       yum: name=make state=present update_cache=yes
       when: ansible_distribution == "CentOS"
 
-    - name: Change the emco directory and run the command make repo
-      command: /usr/bin/make repo
-      register: make_repo
-      args:
-        chdir: /opt/multicloud/deployments/helm/v2/emco
-
-    - debug:
-        var: make_repo.stdout_lines
-
     - name: Change the emco directory and run the command make all
       command: /usr/bin/make all
       register: make_all
@@ -46,7 +37,7 @@
         var: make_all.stdout_lines
 
     - name: Change the emco directory and run the command helm install
-      command: /usr/local/bin/helm install dist/packages/emco-0.1.0.tgz --name emco --namespace emco
+      command: /usr/local/bin/helm install --namespace emco --create-namespace --set emco-tools.fluentd.clusterDomain={{ cluster_name.stdout }} emco dist/packages/emco-0.1.0.tgz
       register: helm_install
       args:
         chdir: /opt/multicloud/deployments/helm/v2/emco
diff --git a/kud/deployment_infra/playbooks/configure-kud.yml b/kud/deployment_infra/playbooks/configure-kud.yml
index 6ac0477d..0e32e69d 100644
--- a/kud/deployment_infra/playbooks/configure-kud.yml
+++ b/kud/deployment_infra/playbooks/configure-kud.yml
@@ -23,11 +23,6 @@
       when: helm_client.rc != 0
       vars:
           kubernetes_helm_ver: "v{{ helm_client_version }}"
-  tasks:
-    - name: Initialize helm client
-      command: helm init -c
-      args:
-        creates: ~/.helm
 
 - hosts: kube-node
   become: yes
diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml
index d6f2f6bc..ddfedbb4 100644
--- a/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml
+++ b/kud/deployment_infra/playbooks/configure-onap4k8s-reset.yml
@@ -15,7 +15,7 @@
         file: kud-vars.yml
 
     - name: Change the onap4k8s directory and run helm delete
-      command: /usr/local/bin/helm delete --purge multicloud-onap8ks
+      command: /usr/local/bin/helm uninstall --namespace onap4k8s-ns multicloud-onap8ks
       register: helm_delete
       args:
         chdir: /opt/multicloud/deployments/helm/onap4k8s
@@ -41,15 +41,6 @@
     - debug:
         var: make_clean.stdout_lines
 
-    - name: Change the onap4k8s directory and make repo-stop
-      command: /usr/bin/make repo-stop
-      register: make_repo_stop
-      args:
-        chdir: /opt/multicloud/deployments/helm/onap4k8s
-
-    - debug:
-        var: make_repo_stop.stdout_lines
-
     - name: clean multicloud-k8s path
       file:
         state: absent
diff --git a/kud/deployment_infra/playbooks/configure-onap4k8s.yml b/kud/deployment_infra/playbooks/configure-onap4k8s.yml
index 11729171..c016cf1c 100644
--- a/kud/deployment_infra/playbooks/configure-onap4k8s.yml
+++ b/kud/deployment_infra/playbooks/configure-onap4k8s.yml
@@ -27,15 +27,6 @@
       yum: name=make state=present update_cache=yes
       when: ansible_distribution == "CentOS"
 
-    - name: Change the onap4k8s directory and run the command make repo
-      command: /usr/bin/make repo
-      register: make_repo
-      args:
-        chdir: /opt/multicloud/deployments/helm/onap4k8s
-
-    - debug:
-        var: make_repo.stdout_lines
-
     - name: Change the onap4k8s directory and run the command make all
       command: /usr/bin/make all
       register: make_all
@@ -46,7 +37,7 @@
         var: make_all.stdout_lines
 
     - name: Change the onap4k8s directory and run the command helm install
-      command: /usr/local/bin/helm install dist/packages/multicloud-k8s-5.0.0.tgz --name multicloud-onap8ks --namespace onap4k8s-ns --set service.type=NodePort
+      command: /usr/local/bin/helm install --namespace onap4k8s-ns --create-namespace --set service.type=NodePort multicloud-onap8ks dist/packages/multicloud-k8s-5.0.0.tgz
       register: helm_install
       args:
         chdir: /opt/multicloud/deployments/helm/onap4k8s
diff --git a/kud/deployment_infra/playbooks/kud-vars.yml b/kud/deployment_infra/playbooks/kud-vars.yml
index 30e54f03..8011c901 100644
--- a/kud/deployment_infra/playbooks/kud-vars.yml
+++ b/kud/deployment_infra/playbooks/kud-vars.yml
@@ -57,8 +57,9 @@ cmk_untaint_required: true
 #cmk_exclusive_mode: packed # choose between: packed, spread, default: packed
 
 go_version: '1.12.5'
-kubespray_version: 2.12.6
-helm_client_version: 2.13.1
+kubespray_version: 2.14.1
+# This matches the helm_version from kubespray defaults
+helm_client_version: 3.2.4
 # kud playbooks not compatible with 2.8.0 - see MULTICLOUD-634
 ansible_version: 2.9.7
 
diff --git a/kud/deployment_infra/playbooks/preconfigure-kubespray.yml b/kud/deployment_infra/playbooks/preconfigure-kubespray.yml
new file mode 100644
index 00000000..78e7eda6
--- /dev/null
+++ b/kud/deployment_infra/playbooks/preconfigure-kubespray.yml
@@ -0,0 +1,19 @@
+---
+# The mitogen module used in kubespray requires python2 on the nodes.
+# On some distributions (i.e. Ubuntu 18.04), the default version of
+# python is python3.
+#
+# When python2 is not present a failure message similar to "bash:
+# /usr/bin/python: No such file or directory" will be reported.
+#
+# Note the use of "strategy: linear" below to temporarily bypass
+# mitogen.
+#
+- name: Install python2
+  hosts: k8s-cluster
+  strategy: linear
+  tasks:
+    - name: Install python2
+      package:
+        name: python
+        state: present
diff --git a/kud/hosting_providers/containerized/installer.sh b/kud/hosting_providers/containerized/installer.sh
index ae16b1dd..6fd7c915 100755
--- a/kud/hosting_providers/containerized/installer.sh
+++ b/kud/hosting_providers/containerized/installer.sh
@@ -36,7 +36,6 @@ function _install_ansible {
     pip install --no-cache-dir ansible==$version
 }
 
-# install_k8s() - Install Kubernetes using kubespray tool
 function install_kubespray {
     echo "Deploying kubernetes"
     version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \
@@ -50,7 +49,6 @@ function install_kubespray {
     _install_ansible
     wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball
     tar -C $dest_folder -xzf $tarball
-    mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
     chown -R root:root $dest_folder/kubespray-$version
     mkdir -p ${local_release_dir}/containers
     rm $tarball
@@ -79,11 +77,14 @@ function install_kubespray {
     fi
 }
 
+# install_k8s() - Install Kubernetes using kubespray tool
 function install_k8s {
-    version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \
-        awk -F ': ' '{print $2}')
     local cluster_name=$1
     ansible-playbook $verbose -i \
+        $kud_inventory $kud_playbooks/preconfigure-kubespray.yml \
+        --become --become-user=root | \
+        tee $cluster_log/setup-kubernetes.log
+    ansible-playbook $verbose -i \
         $kud_inventory $dest_folder/kubespray-$version/cluster.yml \
         -e cluster_name=$cluster_name --become --become-user=root | \
         tee $cluster_log/setup-kubernetes.log
@@ -199,6 +200,9 @@ function install_pkg {
 }
 
 function install_cluster {
+    version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \
+        awk -F ': ' '{print $2}')
+    export ANSIBLE_CONFIG=$dest_folder/kubespray-$version/ansible.cfg
     install_k8s $1
     if [ ${2:+1} ]; then
         echo "install default addons and $2"
diff --git a/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
index 5560dd97..0a2953ce 100644
--- a/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
+++ b/kud/hosting_providers/containerized/inventory/group_vars/k8s-cluster.yml
@@ -49,9 +49,6 @@ kubectl_localhost: true
 local_volumes_enabled: true
 local_volume_provisioner_enabled: true
 
-## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.16.9
-
 # Helm deployment
 helm_enabled: true
 
diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh
index 71e4d8b7..94f3c58c 100755
--- a/kud/hosting_providers/vagrant/installer.sh
+++ b/kud/hosting_providers/vagrant/installer.sh
@@ -102,6 +102,7 @@ function _set_environment_file {
     echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment
     echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment
     echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment
+    echo "export ANSIBLE_CONFIG=${ANSIBLE_CONFIG}" | sudo tee --append /etc/environment
 }
 
 # install_k8s() - Install Kubernetes using kubespray tool
@@ -117,7 +118,6 @@ function install_k8s {
     _install_ansible
     wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball
     sudo tar -C $dest_folder -xzf $tarball
-    sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
     sudo chown -R $USER $dest_folder/kubespray-$version
     sudo mkdir -p ${local_release_dir}/containers
     rm $tarball
@@ -139,6 +139,8 @@ function install_k8s {
     if [[ -n "${https_proxy:-}" ]]; then
         echo "https_proxy: \"$https_proxy\"" | tee --append $kud_inventory_folder/group_vars/all.yml
     fi
+    export ANSIBLE_CONFIG=$dest_folder/kubespray-$version/ansible.cfg
+    ansible-playbook $verbose -i $kud_inventory $kud_playbooks/preconfigure-kubespray.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log
     ansible-playbook $verbose -i $kud_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log
 
     # Configure environment
diff --git a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
index 30fd5c0b..ba79b4b9 100644
--- a/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
+++ b/kud/hosting_providers/vagrant/inventory/group_vars/k8s-cluster.yml
@@ -50,9 +50,6 @@ enable_nodelocaldns: false
 local_volumes_enabled: true
 local_volume_provisioner_enabled: true
 
-## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.16.9
-
 # Helm deployment
 helm_enabled: true