Re-factored resources upload into role

Whole functionality of resource files upload is
implemented in resource-data role instead of
playbook level tasks.
This will make roles easier to test
and make it easier to implement other OS support later.

Issue-ID: OOM-1654

Change-Id: I3d3da9381b0e804f511ee854c41554b924d18883
Signed-off-by: Samuli Silvius <s.silvius@partner.samsung.com>

11 files changed, 175 insertions, 164 deletions

diff --git a/ansible/resources.yml b/ansible/resources.yml
new file mode 100644
index 00000000..8e779567
--- /dev/null
+++ b/ansible/resources.yml
@@ -0,0 +1,6 @@
+---
+- name: Transfer needed resources from resource to infra servers
+  hosts: resources[0], infrastructure
+  serial: 1
+  roles:
+    - resource-data
diff --git a/ansible/roles/resource-data/defaults/main.yml b/ansible/roles/resource-data/defaults/main.yml
new file mode 100644
index 00000000..8a1adfa1
--- /dev/null
+++ b/ansible/roles/resource-data/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+resources_on_nfs: no
diff --git a/ansible/roles/resource-data/tasks/main.yml b/ansible/roles/resource-data/tasks/main.yml
index 51127226..bcdc7dd1 100644
--- a/ansible/roles/resource-data/tasks/main.yml
+++ b/ansible/roles/resource-data/tasks/main.yml
@@ -1,2 +1,31 @@
 ---
-- include_tasks: "{{ transport }}-upload.yml"
+- name: Collect source resources
+  block:
+    - name: Check if source dir and files are present
+      stat:
+        path: "{{ item.source }}"
+      loop:
+        - { source: "{{ resources_dir }}/{{ resources_filename | default('thisdoesnotexists', true) }}",
+            target: "{{ app_data_path }}/{{ resources_filename | default('thisdoesnotexists', true) }}" }
+        - { source: "{{ resources_dir }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}",
+            target: "{{ aux_data_path }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}" }
+      register: source_path
+
+    - name: Create initial resources list of dicts
+      set_fact:
+        to_be_uploaded_resources_list: "{{
+            to_be_uploaded_resources_list | default([]) + [
+            {'file': item.item.target | basename,
+            'destination_dir': item.item.target | dirname } ] }}"
+      loop: "{{ source_path.results }}"
+      when: item.stat.exists
+  when: inventory_hostname in groups.resources
+
+- name: "Upload resource files {{ hostvars[groups.resources.0].to_be_uploaded_resources_list }} to infrastructure"
+  include_tasks: upload_resources.yml
+  vars:
+    resources_source_host: "{{ hostvars[groups.resources.0].ansible_host | default(hostvars[groups.resources.0].inventory_hostname) }}"
+    resources_list_of_dicts: "{{ hostvars[groups.resources.0].to_be_uploaded_resources_list }}"
+  when:
+    - inventory_hostname in groups.infrastructure
+    - hostvars[groups.resources.0].to_be_uploaded_resources_list is defined
diff --git a/ansible/roles/resource-data/tasks/nfs-upload.yml b/ansible/roles/resource-data/tasks/nfs-upload.yml
deleted file mode 100644
index 4b5c18d1..00000000
--- a/ansible/roles/resource-data/tasks/nfs-upload.yml
+++ /dev/null
@@ -1,53 +0,0 @@
----
-- name: Upload resources to infrastructure servers over nfs
-  block:
-    - name: Mount resources
-      mount:
-        path: /tmp/resource_data
-        src: "{{ hostvars[groups.resources.0].ansible_host }}:{{ hostvars[groups.resources.0].resources_dir }}"
-        fstype: nfs
-        state: mounted
-
-    - name: Unarchive resources
-      unarchive:
-        src: "/tmp/resource_data/{{ hostvars[groups.resources.0].resources_filename }}"
-        remote_src: yes
-        dest: "{{ app_data_path }}"
-      when: not resources_data_check.stat.exists
-
-    - name: Unarchive auxiliary resources
-      unarchive:
-        src: "/tmp/resource_data/{{ hostvars[groups.resources.0].aux_resources_filename }}"
-        remote_src: yes
-        dest: "{{ aux_data_path }}"
-      when: >
-        hostvars[groups.resources.0].aux_resources_filename is defined
-        and hostvars[groups.resources.0].aux_resources_filename is not none
-        and aux_data_path is defined and aux_data_path is not none
-        and hostvars[groups.resources.0].aux_file_presence.stat.exists
-        and not aux_resources_data_check.stat.exists
-
-  rescue:
-    - name: Removing the resources data due to an error - so the next run can try again
-      command: /bin/false
-      register: upload_failed
-
-  always:
-    - name: unmount resource dir
-      mount:
-        path: /tmp/resource_data
-        src: "{{ hostvars[groups.resources.0].ansible_host }}:{{hostvars[groups.resources.0].resources_dir }}"
-        fstype: nfs
-        state: absent
-
-    - name: Remove the resource data on error
-      file:
-        path: "{{ app_data_path }}"
-        state: absent
-      when: upload_failed is defined
-
-    - name: Remove the auxilliary resource data on error
-      file:
-        path: "{{ aux_data_path }}"
-        state: absent
-      when: upload_failed is defined
diff --git a/ansible/roles/resource-data/tasks/ssh-upload.yml b/ansible/roles/resource-data/tasks/ssh-upload.yml
deleted file mode 100644
index bc7df37f..00000000
--- a/ansible/roles/resource-data/tasks/ssh-upload.yml
+++ /dev/null
@@ -1,60 +0,0 @@
----
-- name: Upload resources to infrastructure servers over ssh
-  block:
-    - name: Upload ssh private key
-      copy:
-        src: "{{ ansible_ssh_private_key_file }}"
-        dest: /root/.ssh/infra_to_resource.privkey
-        mode: 0600
-        owner: root
-        group: root
-        remote_src: no
-
-    - name: Unarchive resources
-      shell: >
-        ssh -o StrictHostKeyChecking=no -o BatchMode=yes
-        -i /root/.ssh/infra_to_resource.privkey
-        {{ hostvars[groups.resources.0].ansible_host }}
-        'cat "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].resources_filename }}"'
-        | tar -C "{{ app_data_path }}" -xf -
-      args:
-        warn: False
-      when: not resources_data_check.stat.exists
-
-    - name: Unarchive auxiliary resources
-      shell: >
-        ssh -i /root/.ssh/infra_to_resource.privkey
-        {{ hostvars[groups.resources.0].ansible_host }}
-        'cat "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].aux_resources_filename }}"'
-        | tar -C "{{ aux_data_path }}" -xf -
-      when: >
-        hostvars[groups.resources.0].aux_resources_filename is defined
-        and hostvars[groups.resources.0].aux_resources_filename is not none
-        and aux_data_path is defined and aux_data_path is not none
-        and hostvars[groups.resources.0].aux_file_presence.stat.exists
-        and not aux_resources_data_check.stat.exists
-      args:
-        warn: False
-
-  rescue:
-    - name: Removing the resources data due to an error - so the next run can try again
-      command: /bin/false
-      register: upload_failed
-
-  always:
-    - name: Remove the ssh private key
-      file:
-        path: /root/.ssh/infra_to_resource.privkey
-        state: absent
-
-    - name: Remove the resource data on error
-      file:
-        path: "{{ app_data_path }}"
-        state: absent
-      when: upload_failed is defined
-
-    - name: Remove the auxilliary resource data on error
-      file:
-        path: "{{ aux_data_path }}"
-        state: absent
-      when: upload_failed is defined
diff --git a/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml b/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml
new file mode 100644
index 00000000..9baca2fe
--- /dev/null
+++ b/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml
@@ -0,0 +1,34 @@
+---
+#
+# Expected variables
+#   resources_source_host
+#   resources_dir
+#   resource_source_filename
+#   resource_destination_directory
+# Output is upload_failed true/false
+#
+- name: "Upload resource {{ resources_dir }}/{{ resource_source_filename }} to infrastructure servers over nfs"
+  block:
+    - name: Mount resource dir
+      mount:
+        path: /tmp/resource_data
+        src: "{{ resources_source_host }}:{{ resources_dir }}"
+        fstype: nfs
+        state: mounted
+
+    - name: "Unarchive resource {{ resources_dir }}/{{ resource_source_filename }} to {{ resource_destination_directory }} dir on infrastructure servers over nfs"
+      unarchive:
+        src: "/tmp/resource_data/{{ resource_source_filename }}"
+        dest: "{{ resource_destination_directory }}"
+        remote_src: yes
+  rescue:
+    - name: Upload failed
+      set_fact:
+        upload_failed: true
+  always:
+    - name: Unmount resource dir
+      mount:
+        path: /tmp/resource_data
+        src: "{{ resources_source_host }}:{{ resources_dir }}"
+        fstype: nfs
+        state: absent
diff --git a/ansible/roles/resource-data/tasks/unarchive-resource.yml b/ansible/roles/resource-data/tasks/unarchive-resource.yml
new file mode 100644
index 00000000..9eafc220
--- /dev/null
+++ b/ansible/roles/resource-data/tasks/unarchive-resource.yml
@@ -0,0 +1,36 @@
+---
+#
+# Wrapper to pass through following variables
+#   resources_source_host
+#   resources_dir
+#   resource_source_filename
+#   resource_destination_directory
+# And handling target directory creation and possible removal on failure.
+# Idempotence is also handled here as nothing is done if resource_destination_directory
+# was already created.
+#
+# Logically also tranport method selection belongs to here but left it to caller
+# as this is called in a loop causing "package_facts" called many times
+# (not sure if it would matter).
+#
+- name: "Create {{ resource_destination_directory }} directory"
+  file:
+    path: "{{ resource_destination_directory }}"
+    state: directory
+  register: create_destination_dir
+
+- name: "Handle transport of one archive file"
+  block:
+    - name: Re-set upload_failed
+      set_fact:
+        upload_failed: false
+
+    - name: "Unarchive resource {{ resource_source_filename }} from host {{ resources_source_host }}, transport is {{ transport }}"
+      include_tasks: "unarchive-{{ transport }}-resource.yml"
+
+    - name: "Remove the destination directory {{ resource_destination_directory }} on error"
+      file:
+        path: "{{ resource_destination_directory }}"
+        state: absent
+      when: upload_failed
+  when: create_destination_dir.changed
diff --git a/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml b/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml
new file mode 100644
index 00000000..4b1b7d75
--- /dev/null
+++ b/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml
@@ -0,0 +1,51 @@
+---
+#
+# Expected variables
+#   resources_source_host
+#   resources_dir
+#   resource_source_filename
+#   resource_destination_directory
+# Output is upload_failed true/false
+#
+- name: "Upload resource {{ resources_dir }}/{{ resource_source_filename }} to infrastructure servers over ssh"
+  block:
+    - name: Upload ssh private key
+      copy:
+        src: "{{ ansible_ssh_private_key_file }}"
+        dest: /root/.ssh/infra_to_resource.privkey
+        mode: 0600
+        owner: root
+        group: root
+        remote_src: no
+
+    - name: Detect if archive is compressed
+      shell: >
+        ssh -o StrictHostKeyChecking=no
+        -i /root/.ssh/infra_to_resource.privkey
+        {{ resources_source_host }}
+        'file "{{ resources_dir }}/{{ resource_source_filename }}"'
+        | grep "compressed"
+      register: compressed
+
+    - name: Set tar extract options
+      set_fact:
+        tar_extract_options: "{{ '-xzf' if compressed.rc == 0 else '-xf' }}"
+
+    - name: "Unarchive resource {{ resources_dir }}/{{ resource_source_filename }} to {{ resource_destination_directory }} dir on infrastructure servers over ssh"
+      shell: >
+        ssh -o StrictHostKeyChecking=no -o BatchMode=yes
+        -i /root/.ssh/infra_to_resource.privkey
+        {{ resources_source_host }}
+        'cat "{{ resources_dir }}/{{ resource_source_filename }}"'
+        | tar -C "{{ resource_destination_directory }}" "{{ tar_extract_options }}" -
+      args:
+        warn: false
+  rescue:
+    - name: Upload failed
+      set_fact:
+        upload_failed: true
+  always:
+    - name: Remove the ssh private key
+      file:
+        path: /root/.ssh/infra_to_resource.privkey
+        state: absent
diff --git a/ansible/roles/resource-data/tasks/upload_resources.yml b/ansible/roles/resource-data/tasks/upload_resources.yml
new file mode 100644
index 00000000..571bc7d6
--- /dev/null
+++ b/ansible/roles/resource-data/tasks/upload_resources.yml
@@ -0,0 +1,15 @@
+---
+- name: Query package facts to check nfs-utils existence
+  package_facts:
+    manager: auto
+
+- name: Set transport fact to nfs or ssh
+  set_fact:
+    transport: "{{ 'nfs' if resources_on_nfs and 'nfs-utils' in ansible_facts.packages else 'ssh' }}"
+
+- name: "Upload resources to infrastructure servers over {{ transport }}"
+  include_tasks: unarchive-resource.yml
+  vars:
+    resource_source_filename: "{{ item.file }}"
+    resource_destination_directory: "{{ item.destination_dir }}"
+  loop: "{{ resources_list_of_dicts }}"
diff --git a/ansible/site.yml b/ansible/site.yml
index 0df534d0..fbf2c389 100644
--- a/ansible/site.yml
+++ b/ansible/site.yml
@@ -13,7 +13,7 @@
 # 3. playbooks here are more or less batch jobs and the ssh authentication
 #    is a precondition, which should be done during configuration of the
 #    installer
-- import_playbook: upload_resources.yml
+- import_playbook: resources.yml
 - import_playbook: infrastructure.yml
 - import_playbook: rancher_kubernetes.yml
 - import_playbook: application.yml
diff --git a/ansible/upload_resources.yml b/ansible/upload_resources.yml
deleted file mode 100644
index 68010eb1..00000000
--- a/ansible/upload_resources.yml
+++ /dev/null
@@ -1,49 +0,0 @@
----
-- name: Check for presence of auxiliary resources tar file
-  hosts: resources[0]
-  tasks:
-    - name: Store auxiliary resources tar file info into variable
-      stat:
-        path: "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].aux_resources_filename }}"
-      register: aux_file_presence
-
-- name: Check infrastructure server for presence of resources and requirements
-  hosts: infrastructure
-  tasks:
-    - name: Check if nfs-utils is installed
-      yum:
-        list: nfs-utils
-      register: nfs_utils_check
-
-    - name: Check if the resources are already unpacked
-      stat:
-        path: "{{ app_data_path }}"
-      register: resources_data_check
-
-    - name: Check if the auxilliary resources are already unpacked
-      stat:
-        path: "{{ aux_data_path }}"
-      register: aux_resources_data_check
-      when: aux_data_path is defined and aux_data_path is not none
-
-- name: Ensure the existence of data directory/ies on infrastructure server
-  hosts: infrastructure
-  tasks:
-    - name: Create data directory
-      file:
-        path: "{{ app_data_path }}"
-        state: directory
-
-    - name: Create auxiliary data directory
-      file:
-        path: "{{ aux_data_path }}"
-        state: directory
-      when: aux_data_path is defined and aux_data_path is not none
-
-- name: Upload resources to infrastructure server
-  hosts: infrastructure
-  roles:
-    # use nfs or ssh and unpack resources into data directory/ies
-    - role: resource-data
-      vars:
-        transport: "{{ 'nfs' if resources_on_nfs and (nfs_utils_check.results|selectattr('yumstate', 'match', 'installed')|list|length != 0) else 'ssh' }}"