Merge changes from topic "ansible-helm3"

* changes:
  Run chartmuseum as a docker container
  Update 'helm deploy' failure criteria
  Fix kubeconfig file permissions
  Ensure k8s namespace for ONAP exists
  Customize helm "--timeout" option format if running helm v3
  Redirect chartmuseum stdout/stderr to /dev/null
  Drop DIND specific test env settings for 'rke' role/playbook
  Improve bin utils symlink creation logic
  Add test scenario for Helm v3
  Refactor Helm role test setup
  Add helm-push Helm v3 plugin to downloaded utilities list
  Fix Helm v3 data dir setup
  Add tasks to setup Helm v3 on infra node
  Add Helm v3 test scenario to rke playbook tests
  Change 'rke' role testing strategy
  Workaround RKE binary download issue
  Play 'chartmuseum' role in rke playbook if running with Helm v3
  Fix variable inclusion order in RKE playbook tests
  Add Helm v3.3.4 to the list of downloaded utilities
  Add 'chartmuseum' binary to downloaded utilities list
  Add Molecule test scenario to verify 'chartmuseum' role on Ubuntu
  Add 'chartmuseum' role
  Split Helm v2 setup to separate playbook
  Improve 'application' role test coverage

58 files changed, 473 insertions, 230 deletions

diff --git a/ansible/group_vars/infrastructure.yml b/ansible/group_vars/infrastructure.yml
index a1643a11..586f7925 100755
--- a/ansible/group_vars/infrastructure.yml
+++ b/ansible/group_vars/infrastructure.yml
@@ -20,8 +20,10 @@ all_simulated_hosts:
 populate_nexus: false
 helm_bin_dir: /usr/local/bin
 helm_version: v2.16.6
+helm3_push_version: 0.9.0
 rancher_server_image: rancher/server:v1.6.22
 vnc_server_image: consol/ubuntu-icewm-vnc:1.4.0
 nexus3_image: sonatype/nexus3:3.15.2
 dns_server_image: andyshinn/dnsmasq:2.76
 nginx_server_image: own_nginx:2.0.0
+chartmuseum_server_image: chartmuseum/chartmuseum
diff --git a/ansible/infrastructure.yml b/ansible/infrastructure.yml
index 2322c8bb..27d9d76a 100644
--- a/ansible/infrastructure.yml
+++ b/ansible/infrastructure.yml
@@ -16,6 +16,8 @@
     - nginx
     - chrony
     - nexus
+    - role: chartmuseum
+      when: helm_version | regex_search("^v3" )
 
 - name: Setup base for Kubernetes nodes
   hosts: kubernetes:!infrastructure
diff --git a/ansible/roles/application/defaults/main.yml b/ansible/roles/application/defaults/main.yml
index 6a7472a6..8e630f73 100644
--- a/ansible/roles/application/defaults/main.yml
+++ b/ansible/roles/application/defaults/main.yml
@@ -2,7 +2,7 @@
 helm_repository_name: local
 helm_repository_url: http://127.0.0.1:8879
 helm_extra_install_options:
-  - { opt: '--timeout 1800'}
+  - { opt: "{% if helm_version | regex_search('^v3' ) %}{{ '--timeout 1800s' }}{% else %}{{ '--timeout 1800' }}{% endif %}"}
 # Override file generation for Helm application can be customized by any role
 # given by user and found by ansible from roles_path.
 # By default override file is generated by 'application-override' role that
diff --git a/ansible/roles/application/molecule/default/tests/test_default.py b/ansible/roles/application/molecule/default/tests/test_default.py
index 21fc40e4..f57f5afc 100644
--- a/ansible/roles/application/molecule/default/tests/test_default.py
+++ b/ansible/roles/application/molecule/default/tests/test_default.py
@@ -10,21 +10,30 @@ def test_helm_commands(host):
     fc = host.file('/tmp/helm_simu_output').content_string
     helm_release = host.ansible.get_variables()['helm_version']
     if helm_release == 'v2':
-        content_str1 = 'home'
-    elif helm_release == 'v3':
-        content_str1 = 'env'
-    expected_content = content_str1 + """
+        expected_content = """home
 init --upgrade --skip-refresh
 version --tiller-connection-timeout 10
 repo list
 serve
 repo list
 repo add local http://127.0.0.1:8879
-install --name moleculetestapp local/moleculetestapp --namespace \
+deploy moleculetestapp local/moleculetestapp --namespace \
 moleculetestapp -f /opt/moleculetestapp/helm_charts/onap/resources/\
 overrides/onap-all.yaml -f /opt/moleculetestapp/override.yaml \
 --timeout 1800"""
+        expected_plugin_path = '/plugins/deploy/deploy.sh'
+    elif helm_release == 'v3':
+        expected_content = """env
+repo list
+repo add local http://127.0.0.1:8879
+deploy moleculetestapp local/moleculetestapp --namespace \
+moleculetestapp -f /opt/moleculetestapp/helm_charts/onap/resources/\
+overrides/onap-all.yaml -f /opt/moleculetestapp/override.yaml \
+--timeout 1800s"""
+        expected_plugin_path = '/root/.local/share/helm/plugins/deploy/' +\
+                               'deploy.sh'
     assert fc == expected_content
+    assert host.file(expected_plugin_path).exists
 
 
 def test_helm_override_file(host):
diff --git a/ansible/roles/application/tasks/install-helm3-plugins.yml b/ansible/roles/application/tasks/install-helm3-plugins.yml
index da402f31..5d933ed2 100644
--- a/ansible/roles/application/tasks/install-helm3-plugins.yml
+++ b/ansible/roles/application/tasks/install-helm3-plugins.yml
@@ -6,12 +6,12 @@
       register: helm_env
     - name: Set helm data dir
       set_fact:
-        helm_data_dir: |
-          "{% if 'HELM_DATA_HOME' in helm_env.stdout %}
+        helm_data_dir:
+          "{% if 'HELM_DATA_HOME' in helm_env.stdout -%}
            {{ (helm_env.stdout | replace('\"', '') | regex_search('HELM_DATA_HOME.*')).split('=')[1] }}
-           {% else %}
+           {%- else -%}
            {{ '~/.local/share/helm' }}
-           {% endif %}"
+           {%- endif %}"
     - name: Ensure that dir for helm plugins exists
       file:
         path: "{{ helm_data_dir }}/plugins"
diff --git a/ansible/roles/application/tasks/install.yml b/ansible/roles/application/tasks/install.yml
index 9e27e2de..2db88631 100644
--- a/ansible/roles/application/tasks/install.yml
+++ b/ansible/roles/application/tasks/install.yml
@@ -1,51 +1,4 @@
 ---
-- name: Helm init and upgrade
-  command: |
-     {{ helm_bin_dir }}/helm init
-     --upgrade
-     --skip-refresh
-  changed_when: true  # init is always changed type of action
-
-# A correct way to implement this would be using --wait option in helm init invocation.
-# However, it does not work due to https://github.com/helm/helm/issues/4031 (fixed in newer helm release)
-- name: "Wait for helm upgrade to finish"
-  command: "{{ helm_bin_dir }}/helm version --tiller-connection-timeout 10"
-  register: result
-  until: result.rc == 0
-  delay: 10
-  retries: 12
-  changed_when: false  # for idempotency
-
-- name: Get all helm repos
-  command: "{{ helm_bin_dir }}/helm repo list"
-  register: repos
-  changed_when: false  # for idempotency
-
-- name: Remove stable repo
-  command: "{{ helm_bin_dir }}/helm repo remove stable"
-  changed_when: true  # when executed its a changed type of action
-  when: "'stable' in repos.stdout"
-
-- name: Helm Serve
-  shell: "{{ helm_bin_dir }}/helm serve &"
-  async: 45
-  poll: 3  # wait 3sec to get a chance for some stderr
-  register: helm_serve
-  changed_when: "'address already in use' not in helm_serve.stderr"
-
-- name: List helm repos
-  command: "{{ helm_bin_dir }}/helm repo list"
-  register: helm_repo_list
-  changed_when: false  # for idempotency
-  failed_when:
-    - helm_repo_list.rc > 0
-    - "'Error: no repositories to show' not in helm_repo_list.stderr"
-
-- name: Helm Add Repo
-  command: "{{ helm_bin_dir }}/helm repo add {{ helm_repository_name | mandatory }} {{ helm_repository_url | mandatory }}"
-  when: "'local' not in helm_repo_list.stdout"
-  changed_when: true  # when executed its a changed type of action
-
 # Make utility is missing in Ubuntu by default and it's necessary for building local helm repository
 - name: Install build-essential
   package:
@@ -53,6 +6,11 @@
     state: present
   when: ansible_os_family == "Debian"
 
+- include_tasks: setup-helm2.yml
+  when: helm_version | regex_search("^v2" )
+- include_tasks: setup-helm3.yml
+  when: helm_version | regex_search("^v3" )
+
 - name: Build local helm repository
   make:
     chdir: "{{ app_helm_charts_infra_directory }}"
@@ -78,10 +36,11 @@
   debug:
     var: helm_override_files
 
-- include_tasks: setup-helm2.yml
-  when: helm_version | regex_search("^v2" )
-- include_tasks: setup-helm3.yml
-  when: helm_version | regex_search("^v3" )
+- name: "Ensure kubernetes namespace for {{ app_name }} exists"
+  command: kubectl create namespace {{ app_kubernetes_namespace }}
+  register: kubectl_out
+  changed_when: kubectl_out.rc == 0
+  failed_when: kubectl_out.rc == 1 and "AlreadyExists" not in kubectl_out.stderr
 
 - name: "Helm Install application {{ app_name }}"
   command: >
@@ -94,4 +53,4 @@
           {{ helm_extra_install_options | map(attribute='opt') | join(' ') }}
   changed_when: true  # when executed its a changed type of action
   register: helm_install
-  failed_when: helm_install.stderr
+  failed_when: "'FAILED' in (helm_install.stdout | upper()) or helm_install.rc != 0"
diff --git a/ansible/roles/application/tasks/setup-helm2.yml b/ansible/roles/application/tasks/setup-helm2.yml
index 77f0ee91..8479e1a1 100644
--- a/ansible/roles/application/tasks/setup-helm2.yml
+++ b/ansible/roles/application/tasks/setup-helm2.yml
@@ -1,4 +1,51 @@
 ---
+- name: Helm init and upgrade
+  command: |
+     {{ helm_bin_dir }}/helm init
+     --upgrade
+     --skip-refresh
+  changed_when: true  # init is always changed type of action
+
+# A correct way to implement this would be using --wait option in helm init invocation.
+# However, it does not work due to https://github.com/helm/helm/issues/4031 (fixed in newer helm release)
+- name: "Wait for helm upgrade to finish"
+  command: "{{ helm_bin_dir }}/helm version --tiller-connection-timeout 10"
+  register: result
+  until: result.rc == 0
+  delay: 10
+  retries: 12
+  changed_when: false  # for idempotency
+
+- name: Get all helm repos
+  command: "{{ helm_bin_dir }}/helm repo list"
+  register: repos
+  changed_when: false  # for idempotency
+
+- name: Remove stable repo
+  command: "{{ helm_bin_dir }}/helm repo remove stable"
+  changed_when: true  # when executed its a changed type of action
+  when: "'stable' in repos.stdout"
+
+- name: Helm Serve
+  shell: "{{ helm_bin_dir }}/helm serve &"
+  async: 45
+  poll: 3  # wait 3sec to get a chance for some stderr
+  register: helm_serve
+  changed_when: "'address already in use' not in helm_serve.stderr"
+
+- name: List helm repos
+  command: "{{ helm_bin_dir }}/helm repo list"
+  register: helm_repo_list
+  changed_when: false  # for idempotency
+  failed_when:
+    - helm_repo_list.rc > 0
+    - "'Error: no repositories to show' not in helm_repo_list.stderr"
+
+- name: Helm Add Repo
+  command: "{{ helm_bin_dir }}/helm repo add {{ helm_repository_name | mandatory }} {{ helm_repository_url | mandatory }}"
+  when: "'local' not in helm_repo_list.stdout"
+  changed_when: true  # when executed its a changed type of action
+
 - name: Check for deploy plugin presence
   stat:
     path: '{{ helm_home_dir.stdout }}/plugins/deploy/deploy.sh'
diff --git a/ansible/roles/application/tasks/setup-helm3.yml b/ansible/roles/application/tasks/setup-helm3.yml
index ce8cbb3a..4b50ec1c 100644
--- a/ansible/roles/application/tasks/setup-helm3.yml
+++ b/ansible/roles/application/tasks/setup-helm3.yml
@@ -1,4 +1,17 @@
 ---
+- name: List helm repos
+  command: "{{ helm_bin_dir }}/helm repo list"
+  register: helm_repo_list
+  changed_when: false  # for idempotency
+  failed_when:
+    - helm_repo_list.rc > 0
+    - "'Error: no repositories to show' not in helm_repo_list.stderr"
+
+- name: Helm Add Repo
+  command: "{{ helm_bin_dir }}/helm repo add {{ helm_repository_name | mandatory }} {{ helm_repository_url | mandatory }}"
+  when: "'local' not in helm_repo_list.stdout"
+  changed_when: true  # when executed its a changed type of action
+
 - name: Check for deploy plugin presence
   stat:
     path: '{{ helm_data_dir }}/plugins/deploy/deploy.sh'
diff --git a/ansible/roles/chartmuseum/.yamllint b/ansible/roles/chartmuseum/.yamllint
new file mode 100644
index 00000000..ad0be760
--- /dev/null
+++ b/ansible/roles/chartmuseum/.yamllint
@@ -0,0 +1,11 @@
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  line-length: disable
+  truthy: disable
diff --git a/ansible/roles/chartmuseum/defaults/main.yml b/ansible/roles/chartmuseum/defaults/main.yml
new file mode 100644
index 00000000..1d3ae1ba
--- /dev/null
+++ b/ansible/roles/chartmuseum/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+chartmuseum_server_image_tar: "{{ chartmuseum_server_image | regex_replace('(\\/|\\:)', '_') }}.tar"
+chartmuseum_port: "8879"
+chartmuseum_storage_dir: "{{ app_data_path }}/chartmuseum"
diff --git a/ansible/roles/chartmuseum/handlers/main.yml b/ansible/roles/chartmuseum/handlers/main.yml
new file mode 100644
index 00000000..867aaf0f
--- /dev/null
+++ b/ansible/roles/chartmuseum/handlers/main.yml
@@ -0,0 +1,14 @@
+---
+- name: Run chartmuseum server container
+  docker_container:
+    name: chartmuseum
+    image: "{{ chartmuseum_server_image }}"
+    env:
+      STORAGE: "local"
+      STORAGE_LOCAL_ROOTDIR: "/charts"
+    volumes:
+      - "{{ chartmuseum_storage_dir }}:/charts:rw"
+    ports:
+      - "{{ chartmuseum_port }}:8080/tcp"
+    restart_policy: unless-stopped
+    recreate: true
diff --git a/ansible/roles/chartmuseum/molecule/default/molecule.yml b/ansible/roles/chartmuseum/molecule/default/molecule.yml
new file mode 100644
index 00000000..9c5d1c2a
--- /dev/null
+++ b/ansible/roles/chartmuseum/molecule/default/molecule.yml
@@ -0,0 +1,33 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: instance-chartmuseum
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: True
+    privileged: true
+    override_command: False
+    volumes:
+      - /var/lib/docker
+    groups:
+      - infrastructure
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../test/roles
+    ANSIBLE_LIBRARY: ../../../../library
+  inventory:
+    links:
+      group_vars: ../../../../group_vars
+scenario:
+  name: default
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
diff --git a/ansible/roles/chartmuseum/molecule/default/playbook.yml b/ansible/roles/chartmuseum/molecule/default/playbook.yml
new file mode 100644
index 00000000..2694582a
--- /dev/null
+++ b/ansible/roles/chartmuseum/molecule/default/playbook.yml
@@ -0,0 +1,5 @@
+---
+- name: Converge
+  hosts: all
+  roles:
+    - chartmuseum
diff --git a/ansible/roles/chartmuseum/molecule/default/prepare.yml b/ansible/roles/chartmuseum/molecule/default/prepare.yml
new file mode 100644
index 00000000..dee6a976
--- /dev/null
+++ b/ansible/roles/chartmuseum/molecule/default/prepare.yml
@@ -0,0 +1,8 @@
+---
+- name: Prepare infra
+  hosts: infrastructure
+  vars_files:
+    - ../../../../roles/chartmuseum/defaults/main.yml
+  roles:
+    - prepare-docker-dind
+    - prepare-chartmuseum
diff --git a/ansible/roles/chartmuseum/molecule/default/tests/test_default.py b/ansible/roles/chartmuseum/molecule/default/tests/test_default.py
new file mode 100644
index 00000000..40773065
--- /dev/null
+++ b/ansible/roles/chartmuseum/molecule/default/tests/test_default.py
@@ -0,0 +1,14 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_chartmuseum_dir(host):
+    assert host.file("/chartmuseum").is_directory
+
+
+def test_container_running(host):
+    assert host.docker('chartmuseum').is_running
diff --git a/ansible/roles/chartmuseum/molecule/ubuntu/molecule.yml b/ansible/roles/chartmuseum/molecule/ubuntu/molecule.yml
new file mode 100644
index 00000000..5f351a93
--- /dev/null
+++ b/ansible/roles/chartmuseum/molecule/ubuntu/molecule.yml
@@ -0,0 +1,37 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: instance-ubuntu-chartmuseum
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04}
+    pre_build_image: True
+    privileged: true
+    override_command: False
+    volumes:
+      - /var/lib/docker
+    groups:
+      - infrastructure
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../test/roles
+    ANSIBLE_LIBRARY: ../../../../library
+  inventory:
+    links:
+      group_vars: ../../../../group_vars
+  playbooks:
+    prepare: ../default/prepare.yml
+    converge: ../default/playbook.yml
+scenario:
+  name: ubuntu
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+  directory: ../default/tests/
diff --git a/ansible/roles/chartmuseum/tasks/main.yml b/ansible/roles/chartmuseum/tasks/main.yml
new file mode 100644
index 00000000..16e41022
--- /dev/null
+++ b/ansible/roles/chartmuseum/tasks/main.yml
@@ -0,0 +1,14 @@
+---
+- name: Create storage directory for chartmuseum
+  file:
+    path: "{{ chartmuseum_storage_dir }}"
+    state: directory
+    mode: 0777
+
+- name: Load chartmuseum server image
+  docker_image:
+    name: "{{ chartmuseum_server_image }}"
+    load_path: "{{ infra_images_path }}/{{ chartmuseum_server_image_tar }}"
+    state: present
+    timeout: 120
+  notify: Run chartmuseum server container
diff --git a/ansible/roles/helm/molecule/default/Dockerfile.j2 b/ansible/roles/helm/molecule/default/Dockerfile.j2
new file mode 100644
index 00000000..e6aa95d3
--- /dev/null
+++ b/ansible/roles/helm/molecule/default/Dockerfile.j2
@@ -0,0 +1,14 @@
+# Molecule managed
+
+{% if item.registry is defined %}
+FROM {{ item.registry.url }}/{{ item.image }}
+{% else %}
+FROM {{ item.image }}
+{% endif %}
+
+RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
+    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
+    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
+    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
+    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
+    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi
diff --git a/ansible/roles/helm/molecule/default/group_vars/infrastructure.yml b/ansible/roles/helm/molecule/default/group_vars/infrastructure.yml
deleted file mode 120000
index 3e9c2f0c..00000000
--- a/ansible/roles/helm/molecule/default/group_vars/infrastructure.yml
+++ /dev/null
@@ -1 +0,0 @@
-../../../../../group_vars/infrastructure.yml
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/default/molecule.yml b/ansible/roles/helm/molecule/default/molecule.yml
index 0d46c2d4..359d3aba 100644
--- a/ansible/roles/helm/molecule/default/molecule.yml
+++ b/ansible/roles/helm/molecule/default/molecule.yml
@@ -7,10 +7,7 @@ lint:
   name: yamllint
 platforms:
   - name: infrastructure-server
-    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
-    pre_build_image: True
-    privileged: true
-    override_command: False
+    image: centos:7
     groups:
       - infrastructure
 provisioner:
@@ -25,7 +22,6 @@ provisioner:
       all:
         app_name: onap
         app_data_path: "/opt/{{ app_name }}"
-        helm_bin_dir: /usr/local/bin
 scenario:
   name: default
 verifier:
diff --git a/ansible/roles/helm/molecule/default/playbook.yml b/ansible/roles/helm/molecule/default/playbook.yml
index 2705b165..0f3fbc27 100644
--- a/ansible/roles/helm/molecule/default/playbook.yml
+++ b/ansible/roles/helm/molecule/default/playbook.yml
@@ -1,5 +1,10 @@
 ---
 - name: Converge
   hosts: all
+  pre_tasks:
+    - name: Include infrastructure group variables
+      include_vars: ../../../../group_vars/infrastructure.yml
+    - name: Include test scenario variables
+      include_vars: vars.yml
   roles:
     - helm
diff --git a/ansible/roles/helm/molecule/default/prepare.yml b/ansible/roles/helm/molecule/default/prepare.yml
index 34c41e8e..10ccf232 100644
--- a/ansible/roles/helm/molecule/default/prepare.yml
+++ b/ansible/roles/helm/molecule/default/prepare.yml
@@ -4,5 +4,7 @@
   pre_tasks:
     - name: Include infrastructure group variables
       include_vars: ../../../../group_vars/infrastructure.yml
+    - name: Include test scenario variables
+      include_vars: vars.yml
   roles:
     - prepare-helm
diff --git a/ansible/roles/helm/molecule/default/vars.yml b/ansible/roles/helm/molecule/default/vars.yml
new file mode 100644
index 00000000..ed97d539
--- /dev/null
+++ b/ansible/roles/helm/molecule/default/vars.yml
@@ -0,0 +1 @@
+---
diff --git a/ansible/roles/helm/molecule/helm3/Dockerfile.j2 b/ansible/roles/helm/molecule/helm3/Dockerfile.j2
new file mode 120000
index 00000000..867ec5c3
--- /dev/null
+++ b/ansible/roles/helm/molecule/helm3/Dockerfile.j2
@@ -0,0 +1 @@
+../default/Dockerfile.j2
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/helm3/molecule.yml b/ansible/roles/helm/molecule/helm3/molecule.yml
new file mode 100644
index 00000000..e8634477
--- /dev/null
+++ b/ansible/roles/helm/molecule/helm3/molecule.yml
@@ -0,0 +1,31 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: infrastructure-server-helm3
+    image: centos:7
+    groups:
+      - infrastructure
+provisioner:
+  name: ansible
+  lint:
+    name: ansible-lint
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../test/roles
+    ANSIBLE_LIBRARY: ../../../../library
+  inventory:
+    group_vars:
+      all:
+        app_name: onap
+        app_data_path: "/opt/{{ app_name }}"
+scenario:
+  name: helm3
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
+  directory: ../default/tests
diff --git a/ansible/roles/helm/molecule/helm3/playbook.yml b/ansible/roles/helm/molecule/helm3/playbook.yml
new file mode 120000
index 00000000..a3e26797
--- /dev/null
+++ b/ansible/roles/helm/molecule/helm3/playbook.yml
@@ -0,0 +1 @@
+../default/playbook.yml
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/helm3/prepare.yml b/ansible/roles/helm/molecule/helm3/prepare.yml
new file mode 120000
index 00000000..1c017d9a
--- /dev/null
+++ b/ansible/roles/helm/molecule/helm3/prepare.yml
@@ -0,0 +1 @@
+../default/prepare.yml
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/helm3/vars.yml b/ansible/roles/helm/molecule/helm3/vars.yml
new file mode 100644
index 00000000..7ff37715
--- /dev/null
+++ b/ansible/roles/helm/molecule/helm3/vars.yml
@@ -0,0 +1,2 @@
+---
+helm_version: v3.3.4
diff --git a/ansible/roles/helm/molecule/ubuntu/Dockerfile.j2 b/ansible/roles/helm/molecule/ubuntu/Dockerfile.j2
new file mode 120000
index 00000000..867ec5c3
--- /dev/null
+++ b/ansible/roles/helm/molecule/ubuntu/Dockerfile.j2
@@ -0,0 +1 @@
+../default/Dockerfile.j2
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/ubuntu/group_vars b/ansible/roles/helm/molecule/ubuntu/group_vars
deleted file mode 120000
index 5ce8257f..00000000
--- a/ansible/roles/helm/molecule/ubuntu/group_vars
+++ /dev/null
@@ -1 +0,0 @@
-../default/group_vars/
\ No newline at end of file
diff --git a/ansible/roles/helm/molecule/ubuntu/molecule.yml b/ansible/roles/helm/molecule/ubuntu/molecule.yml
index a375a32d..a43ff074 100644
--- a/ansible/roles/helm/molecule/ubuntu/molecule.yml
+++ b/ansible/roles/helm/molecule/ubuntu/molecule.yml
@@ -7,10 +7,7 @@ lint:
   name: yamllint
 platforms:
   - name: infrastructure-server
-    image: molecule-${PREBUILD_PLATFORM_DISTRO:-ubuntu}:${PREBUILD_DISTRO_VERSION:-18.04}
-    pre_build_image: True
-    privileged: true
-    override_command: False
+    image: ubuntu:18.04
     groups:
       - infrastructure
 provisioner:
@@ -28,7 +25,6 @@ provisioner:
       all:
         app_name: onap
         app_data_path: "/opt/{{ app_name }}"
-        helm_bin_dir: /usr/local/bin
 scenario:
   name: ubuntu
 verifier:
diff --git a/ansible/roles/helm/tasks/main.yml b/ansible/roles/helm/tasks/main.yml
index c1b47103..64db7850 100644
--- a/ansible/roles/helm/tasks/main.yml
+++ b/ansible/roles/helm/tasks/main.yml
@@ -9,3 +9,32 @@
       - '*/helm'
     remote_src: true
     mode: 0755
+
+- name: Install helm-push plugin if runing with Helm v3
+  block:
+    - name: Get helm environment information
+      command: "{{ helm_bin_dir }}/helm env"
+      register: helm_env
+      changed_when: false  # for idempotency
+
+    - name: Set helm plugin dir fact
+      set_fact:
+        helm_plugin_dir:
+          "{% if 'HELM_PLUGINS' in helm_env.stdout -%}
+           {{ (helm_env.stdout | replace('\"', '') | regex_search('HELM_PLUGINS.*')).split('=')[1] }}
+           {%- else -%}
+           {{ '~/.local/share/helm/plugins' }}
+           {%- endif %}"
+
+    - name: Ensure that plugin directory exists
+      file:
+        path: "{{ helm_plugin_dir }}/helm-push"
+        state: directory
+        mode: 0755
+
+    - name: Deploy helm-push plugin
+      unarchive:
+        src: "{{ app_data_path }}/downloads/helm-push_{{ helm3_push_version }}_linux_amd64.tar.gz"
+        dest: "{{ helm_plugin_dir }}/helm-push"
+        remote_src: true
+  when: helm_version | regex_search("^v3" )
diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml
index 3078c115..2e25531c 100644
--- a/ansible/roles/rke/molecule/default/molecule.yml
+++ b/ansible/roles/rke/molecule/default/molecule.yml
@@ -11,44 +11,28 @@ platforms:
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    volumes:
-      - /var/lib/kubelet
-      - /var/lib/docker
-    env:
-      container: docker
     groups:
       - infrastructure
       - kubernetes-etcd
       - kubernetes-control-plane
       - kubernetes
-    networks:
-      - name: rke
-    purge_networks: true
-
   - name: kubernetes-node-1
     image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    env:
-      container: docker
-    volumes:
-      - /var/lib/kubelet
-      - /var/lib/docker
     groups:
       - kubernetes
       - kubernetes-node
-    networks:
-      - name: rke
-    purge_networks: true
-
 provisioner:
   name: ansible
   env:
     ANSIBLE_ROLES_PATH: ../../../../test/roles
     ANSIBLE_LIBRARY: ../../../../library
+  inventory:
+    group_vars:
+      all:
+        cluster_config_dir: /opt/onap/cluster
   options:
     e: "app_data_path=/opt/onap"
   lint:
diff --git a/ansible/roles/rke/molecule/default/prepare.yml b/ansible/roles/rke/molecule/default/prepare.yml
index b012790a..d20f1962 100644
--- a/ansible/roles/rke/molecule/default/prepare.yml
+++ b/ansible/roles/rke/molecule/default/prepare.yml
@@ -5,7 +5,6 @@
     - role: prepare-rke
       vars:
         mode: all
-    - prepare-docker-dind
 
 - name: "Infra specific preparations"
   hosts: infrastructure
diff --git a/ansible/roles/rke/molecule/default/tests/test_controlplane.py b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
deleted file mode 100644
index 0bfbca2d..00000000
--- a/ansible/roles/rke/molecule/default/tests/test_controlplane.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import os
-import pytest
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts(
-      'kubernetes-control-plane')
-
-
-@pytest.mark.parametrize('container_name', [
-  'kube-apiserver', 'kube-controller-manager', 'kube-scheduler', 'kubelet'])
-def test_container_running(host, container_name):
-    assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/default/tests/test_etcd.py b/ansible/roles/rke/molecule/default/tests/test_etcd.py
deleted file mode 100644
index 0f4b6f12..00000000
--- a/ansible/roles/rke/molecule/default/tests/test_etcd.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import os
-import pytest
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes-etcd')
-
-
-@pytest.mark.parametrize('container_name', [
-  'etcd'])
-def test_container_running(host, container_name):
-    assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/default/tests/test_infrastructure.py b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
index 731f38bf..81f90d17 100644
--- a/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
+++ b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
@@ -1,6 +1,5 @@
 import os
 import pytest
-import json
 
 import testinfra.utils.ansible_runner
 
@@ -11,7 +10,9 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
 @pytest.mark.parametrize('filename', [
   '/root/.kube/config',
   '/opt/onap/cluster/cluster.yml',
-  '/opt/onap/cluster/cluster.rkestate'])
+  '/opt/onap/cluster/kubernetes-dashboard.yml',
+  '/opt/onap/cluster/k8s-dashboard-user.yml',
+  '/opt/onap/cluster/kube_config_cluster.yml'])
 def test_file_existence(host, filename):
     assert host.file(filename).exists
 
@@ -22,35 +23,4 @@ def test_rke_in_path(host):
 
 def test_rke_version_works(host):
     # Note that we need to cd to the cluster data dir first, really.
-    assert host.run('cd /opt/onap/cluster && rke version').rc == 0
-
-
-def test_nodes_ready(host):
-    # Retrieve all node names.
-    nodecmdres = host.run('kubectl get nodes -o name')
-    assert nodecmdres.rc == 0
-    nodes = nodecmdres.stdout.split('\n')
-    for node in nodes:
-        assert host.run(
-          'kubectl wait --timeout=0 --for=condition=ready ' + node).rc == 0
-
-
-def test_pods_ready(host):
-    # Retrieve all pods from all namespaces.
-    # Because we need pod and namespace name, we get full json representation.
-    podcmdres = host.run('kubectl get pods --all-namespaces -o json')
-    assert podcmdres.rc == 0
-    pods = json.loads(podcmdres.stdout)['items']
-    for pod in pods:
-        # Each pod may be either created by a job or not.
-        # In job case they should already be completed
-        # when we are here so we ignore them.
-        namespace = pod['metadata']['namespace']
-        podname = pod['metadata']['name']
-        condition = 'Ready'
-        if len(pod['metadata']['ownerReferences']) == 1 and pod[
-          'metadata']['ownerReferences'][0]['kind'] == 'Job':
-            continue
-        assert host.run(
-          'kubectl wait --timeout=240s --for=condition=' + condition + ' -n ' +
-          namespace + ' pods/' + podname).rc == 0
+    assert host.run('cd /opt/onap/cluster && rke').rc == 0
diff --git a/ansible/roles/rke/molecule/default/tests/test_nodes.py b/ansible/roles/rke/molecule/default/tests/test_nodes.py
deleted file mode 100644
index 60413018..00000000
--- a/ansible/roles/rke/molecule/default/tests/test_nodes.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import os
-import pytest
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes-node')
-
-
-@pytest.mark.parametrize('container_name', [
-  'kubelet', 'kube-proxy'])
-def test_container_running(host, container_name):
-    assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/etcd_storage/molecule.yml b/ansible/roles/rke/molecule/etcd_storage/molecule.yml
index 3c915284..bff115e5 100644
--- a/ansible/roles/rke/molecule/etcd_storage/molecule.yml
+++ b/ansible/roles/rke/molecule/etcd_storage/molecule.yml
@@ -11,39 +11,19 @@ platforms:
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    volumes:
-      - /var/lib/kubelet
-      - /var/lib/docker
-    env:
-      container: docker
     groups:
       - infrastructure
       - kubernetes-etcd
       - kubernetes-control-plane
       - kubernetes
-    networks:
-      - name: rke
-    purge_networks: true
-
   - name: kubernetes-node-1
     image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    env:
-      container: docker
-    volumes:
-      - /var/lib/kubelet
-      - /var/lib/docker
     groups:
       - kubernetes
       - kubernetes-node
-    networks:
-      - name: rke
-    purge_networks: true
-
 provisioner:
   name: ansible
   env:
@@ -52,18 +32,19 @@ provisioner:
   inventory:
     group_vars:
       all:
+        cluster_config_dir: /opt/onap/cluster
         rke_etcd:
           enabled_custom_etcd_storage: false
           storage_path: /var/lib/etcd-custom
           storage_mountpoint: /var/lib/rancher/etcd-custom
           enabled_unsafe_volatile_storage: true
           tmpfs_size: 5G
+  options:
+    e: "app_data_path=/opt/onap"
   playbooks:
     prepare: ../default/prepare.yml
     converge: ../default/playbook.yml
     destroy: ../default/destroy.yml
-  options:
-    e: "app_data_path=/opt/onap"
   lint:
     name: ansible-lint
 scenario:
diff --git a/ansible/roles/rke/tasks/rke_deploy.yml b/ansible/roles/rke/tasks/rke_deploy.yml
index 7b36f55d..17861ad2 100644
--- a/ansible/roles/rke/tasks/rke_deploy.yml
+++ b/ansible/roles/rke/tasks/rke_deploy.yml
@@ -23,4 +23,4 @@
     src: "{{ cluster_config_dir }}/kube_config_cluster.yml"
     dest: "{{ kube_config_dir }}/config"
     remote_src: true
-    mode: 0755
+    mode: 0600
diff --git a/ansible/test/bin/install-molecule.sh b/ansible/test/bin/install-molecule.sh
index ab6de436..4df06684 100755
--- a/ansible/test/bin/install-molecule.sh
+++ b/ansible/test/bin/install-molecule.sh
@@ -41,5 +41,6 @@ source ${VENV_PATH}/bin/activate
 # Install Molecule
 if [ ! -z ${VIRTUAL_ENV} ]; then
     echo "Activated virtual env in ${VIRTUAL_ENV}"
-    pip -q install molecule==2.20 ansible==2.7.8 ansible-lint==4.2.0 docker pyopenssl
+    pip -q install -U pip
+    pip -q install pyopenssl molecule==2.20 ansible==2.7.8 ansible-lint==4.2.0 docker
 fi
diff --git a/ansible/test/play-infrastructure/molecule/default/prepare.yml b/ansible/test/play-infrastructure/molecule/default/prepare.yml
index 86e25b48..634234ac 100644
--- a/ansible/test/play-infrastructure/molecule/default/prepare.yml
+++ b/ansible/test/play-infrastructure/molecule/default/prepare.yml
@@ -9,12 +9,14 @@
     - prepare-nexus
     - prepare-dns
     - prepare-firewall
+    - prepare-chartmuseum
   vars_files:
     - vars.yml
     - ../../../../roles/nginx/defaults/main.yml
     - ../../../../roles/vncserver/defaults/main.yml
     - ../../../../roles/nexus/defaults/main.yml
     - ../../../../roles/dns/defaults/main.yml
+    - ../../../../roles/chartmuseum/defaults/main.yml
 
 - name: Prepare kubernetes
   hosts: kubernetes
diff --git a/ansible/test/play-rke/molecule/default/molecule.yml b/ansible/test/play-rke/molecule/default/molecule.yml
index b7e71bf5..519679a5 100644
--- a/ansible/test/play-rke/molecule/default/molecule.yml
+++ b/ansible/test/play-rke/molecule/default/molecule.yml
@@ -11,35 +11,19 @@ platforms:
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    volumes:
-      - /var/lib/docker
-      - /var/lib/kubelet
     groups:
       - infrastructure
       - kubernetes-etcd
       - kubernetes-control-plane
       - kubernetes
-    networks:
-      - name: rke
-    purge_networks: true
-
   - name: kubernetes-node-1
     image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
     pre_build_image: true
     privileged: true
     override_command: false
-    restart_policy: unless-stopped
-    volumes:
-      - /var/lib/docker
-      - /var/lib/kubelet
     groups:
       - kubernetes
       - kubernetes-node
-    networks:
-      - name: rke
-    purge_networks: true
-
 provisioner:
   name: ansible
   env:
diff --git a/ansible/test/play-rke/molecule/default/prepare.yml b/ansible/test/play-rke/molecule/default/prepare.yml
index ad6f0495..6d810052 100644
--- a/ansible/test/play-rke/molecule/default/prepare.yml
+++ b/ansible/test/play-rke/molecule/default/prepare.yml
@@ -2,7 +2,6 @@
 - name: Prepare for all
   hosts: all
   roles:
-    - prepare-docker-dind
     - role: prepare-rke
       vars:
         mode: all
@@ -12,11 +11,11 @@
   pre_tasks:
     - name: Include infrastructure group variables
       include_vars: ../../../../group_vars/infrastructure.yml
+    - name: Include test setup variables
+      include_vars: vars.yml
   roles:
     - role: prepare-rke
       vars:
         mode: infra
     - prepare-kubectl
     - prepare-helm
-  vars_files:
-    vars.yml
diff --git a/ansible/test/play-rke/molecule/default/vars.yml b/ansible/test/play-rke/molecule/default/vars.yml
index fa5b60e3..739dc955 100644
--- a/ansible/test/play-rke/molecule/default/vars.yml
+++ b/ansible/test/play-rke/molecule/default/vars.yml
@@ -1,3 +1,4 @@
 ---
 app_name: moleculeapp
 app_data_path: "/opt/{{ app_name }}"
+cluster_config_dir: "/opt/{{ app_name }}/cluster"
diff --git a/ansible/test/play-rke/molecule/helm3/molecule.yml b/ansible/test/play-rke/molecule/helm3/molecule.yml
new file mode 100644
index 00000000..83a2fd20
--- /dev/null
+++ b/ansible/test/play-rke/molecule/helm3/molecule.yml
@@ -0,0 +1,39 @@
+---
+dependency:
+  name: galaxy
+driver:
+  name: docker
+lint:
+  name: yamllint
+platforms:
+  - name: infrastructure-server
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: true
+    privileged: true
+    override_command: false
+    groups:
+      - infrastructure
+      - kubernetes-etcd
+      - kubernetes-control-plane
+      - kubernetes
+  - name: kubernetes-node-1
+    image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+    pre_build_image: true
+    privileged: true
+    override_command: false
+    groups:
+      - kubernetes
+      - kubernetes-node
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../test/roles
+    ANSIBLE_LIBRARY: ../../../../library
+  lint:
+    name: ansible-lint
+scenario:
+  name: helm3
+verifier:
+  name: testinfra
+  lint:
+    name: flake8
diff --git a/ansible/test/play-rke/molecule/helm3/playbook.yml b/ansible/test/play-rke/molecule/helm3/playbook.yml
new file mode 120000
index 00000000..a3e26797
--- /dev/null
+++ b/ansible/test/play-rke/molecule/helm3/playbook.yml
@@ -0,0 +1 @@
+../default/playbook.yml
\ No newline at end of file
diff --git a/ansible/test/play-rke/molecule/helm3/prepare.yml b/ansible/test/play-rke/molecule/helm3/prepare.yml
new file mode 120000
index 00000000..1c017d9a
--- /dev/null
+++ b/ansible/test/play-rke/molecule/helm3/prepare.yml
@@ -0,0 +1 @@
+../default/prepare.yml
\ No newline at end of file
diff --git a/ansible/test/play-rke/molecule/helm3/vars.yml b/ansible/test/play-rke/molecule/helm3/vars.yml
new file mode 100644
index 00000000..e16e502f
--- /dev/null
+++ b/ansible/test/play-rke/molecule/helm3/vars.yml
@@ -0,0 +1,5 @@
+---
+app_name: moleculeapp
+app_data_path: "/opt/{{ app_name }}"
+cluster_config_dir: "/opt/{{ app_name }}/cluster"
+helm_version: v3.3.4
diff --git a/ansible/test/roles/prepare-application/tasks/main.yml b/ansible/test/roles/prepare-application/tasks/main.yml
index 75abb802..9eb695b1 100644
--- a/ansible/test/roles/prepare-application/tasks/main.yml
+++ b/ansible/test/roles/prepare-application/tasks/main.yml
@@ -6,6 +6,7 @@
   delegate_to: localhost
   loop:
     - "{{ app_helm_charts_install_directory }}"
+    - "{{ app_helm_charts_install_directory + '/helm' + '/plugins' + '/deploy' }}"
     - certs
     - "{{ application_pre_install_role + '/tasks/' }}"
     - "{{ application_post_install_role + '/tasks/' }}"
@@ -19,6 +20,14 @@
   delegate_to: localhost
   when: simulate_helm
 
+- name: Create Helm plugin mock
+  copy:
+    content: |
+      # Mocked Helm plugin
+    dest: "{{ app_helm_charts_install_directory + '/helm' + '/plugins' + '/deploy' }}/deploy.sh"
+  delegate_to: localhost
+  when: simulate_helm
+
 - name: Clean previous simulation output file
   file:
     path: "{{ helm_simulation_output_file }}"
diff --git a/ansible/test/roles/prepare-chartmuseum/tasks/main.yml b/ansible/test/roles/prepare-chartmuseum/tasks/main.yml
new file mode 100644
index 00000000..a46888e8
--- /dev/null
+++ b/ansible/test/roles/prepare-chartmuseum/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+- name: "Create chartmuseum image storage dir"
+  file:
+    path: "{{ infra_images_path }}"
+    state: directory
+    mode: "u+rw,g+wx,o+rwx"
+    recurse: true
+
+- name: Download and archive chartmuseum docker image
+  delegate_to: localhost
+  docker_image:
+    name: "{{ chartmuseum_server_image }}"
+    archive_path: /tmp/chartmuseum.tar
+
+- name: Copy chartmuseum image to node
+  copy:
+    src: /tmp/chartmuseum.tar
+    dest: "{{ infra_images_path }}/{{ chartmuseum_server_image_tar }}"
diff --git a/ansible/test/roles/prepare-helm/tasks/main.yml b/ansible/test/roles/prepare-helm/tasks/main.yml
index 1f461258..c1d25de0 100644
--- a/ansible/test/roles/prepare-helm/tasks/main.yml
+++ b/ansible/test/roles/prepare-helm/tasks/main.yml
@@ -9,4 +9,13 @@
   get_url:
     url: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz"
     dest: "{{ app_data_path }}/downloads"
-    remote_src: true
+
+- name: Download Helm v3 helm-push plugin
+  command: "curl --connect-timeout 10 -L https://github.com/chartmuseum/helm-push/releases/download/v{{ helm3_push_version }}/helm-push_{{ helm3_push_version }}_linux_amd64.tar.gz -o {{ app_data_path }}/downloads/helm-push_{{ helm3_push_version }}_linux_amd64.tar.gz"
+  register: result
+  retries: 10
+  delay: 1
+  until: not result.failed
+  args:
+    warn: false
+  when: helm_version | regex_search("^v3" )
diff --git a/ansible/test/roles/prepare-rke/tasks/all.yml b/ansible/test/roles/prepare-rke/tasks/all.yml
index d4b67c1f..a259577c 100644
--- a/ansible/test/roles/prepare-rke/tasks/all.yml
+++ b/ansible/test/roles/prepare-rke/tasks/all.yml
@@ -1,6 +1,4 @@
-#This is needed because login from non root is blocked by default.
-- name: "Allow non root logins"
-  service:
-    name: systemd-user-sessions
-    state: started
-
+---
+- name: "Add docker group"
+  group:
+    name: docker
diff --git a/ansible/test/roles/prepare-rke/tasks/infra.yml b/ansible/test/roles/prepare-rke/tasks/infra.yml
index 6e7bcb96..0a0b9273 100644
--- a/ansible/test/roles/prepare-rke/tasks/infra.yml
+++ b/ansible/test/roles/prepare-rke/tasks/infra.yml
@@ -4,7 +4,22 @@
     path: "{{ app_data_path }}/downloads"
     state: directory
 
-- name: "Install rke-{{ rke_version }}"
-  get_url:
-    url: "https://github.com/rancher/rke/releases/download/v{{ rke_version }}/rke_linux-amd64"
+- name: "Create rke binary wrapper"
+  template:
+    src: rke.j2
     dest: "{{ app_data_path }}/downloads/rke_linux-amd64"
+    mode: 0755
+
+- name: "Install rke-{{ rke_version }}"
+  command: "curl --connect-timeout 10 -L https://github.com/rancher/rke/releases/download/v{{ rke_version }}/rke_linux-amd64 -o {{ app_data_path }}/downloads/rke_linux-amd64_real"
+  register: result
+  retries: 10
+  delay: 1
+  until: not result.failed
+  args:
+    warn: false
+
+- name: "Ensure downloaded rke binary has correct file mode"
+  file:
+    path: "{{ app_data_path }}/downloads/rke_linux-amd64_real"
+    mode: 0755
diff --git a/ansible/test/roles/prepare-rke/templates/rke.j2 b/ansible/test/roles/prepare-rke/templates/rke.j2
new file mode 100644
index 00000000..ea1267ea
--- /dev/null
+++ b/ansible/test/roles/prepare-rke/templates/rke.j2
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [[ $@ =~ 'up' ]];
+then
+  # If the operation is 'rke up' then simulate rke up operation
+  echo $@
+  touch {{ cluster_config_dir }}/kube_config_cluster.yml
+elif [[ $@ =~ 'version' ]];
+then
+  {{ app_data_path }}/downloads/rke_linux-amd64_real version
+  exit 0
+else
+  {{ app_data_path }}/downloads/rke_linux-amd64_real $@
+fi
diff --git a/build/data_lists/infra_bin_utils.list b/build/data_lists/infra_bin_utils.list
index 36a9fe6b..3beb2972 100644
--- a/build/data_lists/infra_bin_utils.list
+++ b/build/data_lists/infra_bin_utils.list
@@ -1,3 +1,5 @@
 https://storage.googleapis.com/kubernetes-release/release/v1.15.11/bin/linux/amd64/kubectl
 https://get.helm.sh/helm-v2.16.6-linux-amd64.tar.gz
+https://get.helm.sh/helm-v3.3.4-linux-amd64.tar.gz
+https://github.com/chartmuseum/helm-push/releases/download/v0.9.0/helm-push_0.9.0_linux_amd64.tar.gz
 https://github.com/rancher/rke/releases/download/v1.0.4/rke_linux-amd64
diff --git a/build/data_lists/infra_docker_images.list b/build/data_lists/infra_docker_images.list
index 78081cc8..43b2f865 100644
--- a/build/data_lists/infra_docker_images.list
+++ b/build/data_lists/infra_docker_images.list
@@ -1,3 +1,4 @@
 andyshinn/dnsmasq:2.76
 consol/ubuntu-icewm-vnc:1.4.0
 sonatype/nexus3:3.15.2
+chartmuseum/chartmuseum
diff --git a/build/package.py b/build/package.py
index c3d89d30..ce603be5 100755
--- a/build/package.py
+++ b/build/package.py
@@ -243,14 +243,14 @@ def build_offline_deliverables(build_version,
             if os.path.islink(file):
                 os.unlink(file)
 
-        rke_files = glob.glob(os.path.join('.', '**/rke_linux-amd64'), recursive=True)
-        os.symlink(rke_files[0], os.path.join(download_dir_path, rke_files[0].split('/')[-1]))
-
-        helm_tar_files = glob.glob(os.path.join('.', '**/helm-*-linux-amd64.tar.gz'), recursive=True)
-        os.symlink(helm_tar_files[0], os.path.join(download_dir_path, helm_tar_files[0].split('/')[-1]))
-
-        kubectl_files = glob.glob(os.path.join('.', '**/kubectl'), recursive=True)
-        os.symlink(kubectl_files[0], os.path.join(download_dir_path, kubectl_files[0].split('/')[-1]))
+        bin_pattern_list = ['**/rke_linux-amd64',
+                            '**/helm-*-linux-amd64.tar.gz',
+                            '**/kubectl',
+                            '**/helm-push_*_linux_amd64.tar.gz']
+
+        for pattern in bin_pattern_list:
+            for bin_file in glob.glob(os.path.join('.', pattern), recursive=True):
+                os.symlink(bin_file, os.path.join(download_dir_path, bin_file.split('/')[-1]))
 
         os.chdir(script_location)
         # End of workaround