From 29886f09d53bee1ecdc1bc8612e8edb4a130a270 Mon Sep 17 00:00:00 2001 From: Samuli Silvius Date: Tue, 19 Feb 2019 16:00:37 +0200 Subject: Re-factored resources upload into role Whole functionality of resource files upload is implemented in resource-data role instead of playbook level tasks. This will make roles easier to test and make it easier to implement other OS support later. Issue-ID: OOM-1654 Change-Id: I3d3da9381b0e804f511ee854c41554b924d18883 Signed-off-by: Samuli Silvius --- ansible/resources.yml | 6 +++ ansible/roles/resource-data/defaults/main.yml | 2 + ansible/roles/resource-data/tasks/main.yml | 31 ++++++++++- ansible/roles/resource-data/tasks/nfs-upload.yml | 53 ------------------- ansible/roles/resource-data/tasks/ssh-upload.yml | 60 ---------------------- .../resource-data/tasks/unarchive-nfs-resource.yml | 34 ++++++++++++ .../resource-data/tasks/unarchive-resource.yml | 36 +++++++++++++ .../resource-data/tasks/unarchive-ssh-resource.yml | 51 ++++++++++++++++++ .../roles/resource-data/tasks/upload_resources.yml | 15 ++++++ ansible/site.yml | 2 +- ansible/upload_resources.yml | 49 ------------------ 11 files changed, 175 insertions(+), 164 deletions(-) create mode 100644 ansible/resources.yml create mode 100644 ansible/roles/resource-data/defaults/main.yml delete mode 100644 ansible/roles/resource-data/tasks/nfs-upload.yml delete mode 100644 ansible/roles/resource-data/tasks/ssh-upload.yml create mode 100644 ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml create mode 100644 ansible/roles/resource-data/tasks/unarchive-resource.yml create mode 100644 ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml create mode 100644 ansible/roles/resource-data/tasks/upload_resources.yml delete mode 100644 ansible/upload_resources.yml (limited to 'ansible') diff --git a/ansible/resources.yml b/ansible/resources.yml new file mode 100644 index 00000000..8e779567 --- /dev/null +++ b/ansible/resources.yml @@ -0,0 +1,6 @@ +--- +- name: Transfer needed resources from resource to infra servers + hosts: resources[0], infrastructure + serial: 1 + roles: + - resource-data diff --git a/ansible/roles/resource-data/defaults/main.yml b/ansible/roles/resource-data/defaults/main.yml new file mode 100644 index 00000000..8a1adfa1 --- /dev/null +++ b/ansible/roles/resource-data/defaults/main.yml @@ -0,0 +1,2 @@ +--- +resources_on_nfs: no diff --git a/ansible/roles/resource-data/tasks/main.yml b/ansible/roles/resource-data/tasks/main.yml index 51127226..bcdc7dd1 100644 --- a/ansible/roles/resource-data/tasks/main.yml +++ b/ansible/roles/resource-data/tasks/main.yml @@ -1,2 +1,31 @@ --- -- include_tasks: "{{ transport }}-upload.yml" +- name: Collect source resources + block: + - name: Check if source dir and files are present + stat: + path: "{{ item.source }}" + loop: + - { source: "{{ resources_dir }}/{{ resources_filename | default('thisdoesnotexists', true) }}", + target: "{{ app_data_path }}/{{ resources_filename | default('thisdoesnotexists', true) }}" } + - { source: "{{ resources_dir }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}", + target: "{{ aux_data_path }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}" } + register: source_path + + - name: Create initial resources list of dicts + set_fact: + to_be_uploaded_resources_list: "{{ + to_be_uploaded_resources_list | default([]) + [ + {'file': item.item.target | basename, + 'destination_dir': item.item.target | dirname } ] }}" + loop: "{{ source_path.results }}" + when: item.stat.exists + when: inventory_hostname in groups.resources + +- name: "Upload resource files {{ hostvars[groups.resources.0].to_be_uploaded_resources_list }} to infrastructure" + include_tasks: upload_resources.yml + vars: + resources_source_host: "{{ hostvars[groups.resources.0].ansible_host | default(hostvars[groups.resources.0].inventory_hostname) }}" + resources_list_of_dicts: "{{ hostvars[groups.resources.0].to_be_uploaded_resources_list }}" + when: + - inventory_hostname in groups.infrastructure + - hostvars[groups.resources.0].to_be_uploaded_resources_list is defined diff --git a/ansible/roles/resource-data/tasks/nfs-upload.yml b/ansible/roles/resource-data/tasks/nfs-upload.yml deleted file mode 100644 index 4b5c18d1..00000000 --- a/ansible/roles/resource-data/tasks/nfs-upload.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: Upload resources to infrastructure servers over nfs - block: - - name: Mount resources - mount: - path: /tmp/resource_data - src: "{{ hostvars[groups.resources.0].ansible_host }}:{{ hostvars[groups.resources.0].resources_dir }}" - fstype: nfs - state: mounted - - - name: Unarchive resources - unarchive: - src: "/tmp/resource_data/{{ hostvars[groups.resources.0].resources_filename }}" - remote_src: yes - dest: "{{ app_data_path }}" - when: not resources_data_check.stat.exists - - - name: Unarchive auxiliary resources - unarchive: - src: "/tmp/resource_data/{{ hostvars[groups.resources.0].aux_resources_filename }}" - remote_src: yes - dest: "{{ aux_data_path }}" - when: > - hostvars[groups.resources.0].aux_resources_filename is defined - and hostvars[groups.resources.0].aux_resources_filename is not none - and aux_data_path is defined and aux_data_path is not none - and hostvars[groups.resources.0].aux_file_presence.stat.exists - and not aux_resources_data_check.stat.exists - - rescue: - - name: Removing the resources data due to an error - so the next run can try again - command: /bin/false - register: upload_failed - - always: - - name: unmount resource dir - mount: - path: /tmp/resource_data - src: "{{ hostvars[groups.resources.0].ansible_host }}:{{hostvars[groups.resources.0].resources_dir }}" - fstype: nfs - state: absent - - - name: Remove the resource data on error - file: - path: "{{ app_data_path }}" - state: absent - when: upload_failed is defined - - - name: Remove the auxilliary resource data on error - file: - path: "{{ aux_data_path }}" - state: absent - when: upload_failed is defined diff --git a/ansible/roles/resource-data/tasks/ssh-upload.yml b/ansible/roles/resource-data/tasks/ssh-upload.yml deleted file mode 100644 index bc7df37f..00000000 --- a/ansible/roles/resource-data/tasks/ssh-upload.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- name: Upload resources to infrastructure servers over ssh - block: - - name: Upload ssh private key - copy: - src: "{{ ansible_ssh_private_key_file }}" - dest: /root/.ssh/infra_to_resource.privkey - mode: 0600 - owner: root - group: root - remote_src: no - - - name: Unarchive resources - shell: > - ssh -o StrictHostKeyChecking=no -o BatchMode=yes - -i /root/.ssh/infra_to_resource.privkey - {{ hostvars[groups.resources.0].ansible_host }} - 'cat "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].resources_filename }}"' - | tar -C "{{ app_data_path }}" -xf - - args: - warn: False - when: not resources_data_check.stat.exists - - - name: Unarchive auxiliary resources - shell: > - ssh -i /root/.ssh/infra_to_resource.privkey - {{ hostvars[groups.resources.0].ansible_host }} - 'cat "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].aux_resources_filename }}"' - | tar -C "{{ aux_data_path }}" -xf - - when: > - hostvars[groups.resources.0].aux_resources_filename is defined - and hostvars[groups.resources.0].aux_resources_filename is not none - and aux_data_path is defined and aux_data_path is not none - and hostvars[groups.resources.0].aux_file_presence.stat.exists - and not aux_resources_data_check.stat.exists - args: - warn: False - - rescue: - - name: Removing the resources data due to an error - so the next run can try again - command: /bin/false - register: upload_failed - - always: - - name: Remove the ssh private key - file: - path: /root/.ssh/infra_to_resource.privkey - state: absent - - - name: Remove the resource data on error - file: - path: "{{ app_data_path }}" - state: absent - when: upload_failed is defined - - - name: Remove the auxilliary resource data on error - file: - path: "{{ aux_data_path }}" - state: absent - when: upload_failed is defined diff --git a/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml b/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml new file mode 100644 index 00000000..9baca2fe --- /dev/null +++ b/ansible/roles/resource-data/tasks/unarchive-nfs-resource.yml @@ -0,0 +1,34 @@ +--- +# +# Expected variables +# resources_source_host +# resources_dir +# resource_source_filename +# resource_destination_directory +# Output is upload_failed true/false +# +- name: "Upload resource {{ resources_dir }}/{{ resource_source_filename }} to infrastructure servers over nfs" + block: + - name: Mount resource dir + mount: + path: /tmp/resource_data + src: "{{ resources_source_host }}:{{ resources_dir }}" + fstype: nfs + state: mounted + + - name: "Unarchive resource {{ resources_dir }}/{{ resource_source_filename }} to {{ resource_destination_directory }} dir on infrastructure servers over nfs" + unarchive: + src: "/tmp/resource_data/{{ resource_source_filename }}" + dest: "{{ resource_destination_directory }}" + remote_src: yes + rescue: + - name: Upload failed + set_fact: + upload_failed: true + always: + - name: Unmount resource dir + mount: + path: /tmp/resource_data + src: "{{ resources_source_host }}:{{ resources_dir }}" + fstype: nfs + state: absent diff --git a/ansible/roles/resource-data/tasks/unarchive-resource.yml b/ansible/roles/resource-data/tasks/unarchive-resource.yml new file mode 100644 index 00000000..9eafc220 --- /dev/null +++ b/ansible/roles/resource-data/tasks/unarchive-resource.yml @@ -0,0 +1,36 @@ +--- +# +# Wrapper to pass through following variables +# resources_source_host +# resources_dir +# resource_source_filename +# resource_destination_directory +# And handling target directory creation and possible removal on failure. +# Idempotence is also handled here as nothing is done if resource_destination_directory +# was already created. +# +# Logically also tranport method selection belongs to here but left it to caller +# as this is called in a loop causing "package_facts" called many times +# (not sure if it would matter). +# +- name: "Create {{ resource_destination_directory }} directory" + file: + path: "{{ resource_destination_directory }}" + state: directory + register: create_destination_dir + +- name: "Handle transport of one archive file" + block: + - name: Re-set upload_failed + set_fact: + upload_failed: false + + - name: "Unarchive resource {{ resource_source_filename }} from host {{ resources_source_host }}, transport is {{ transport }}" + include_tasks: "unarchive-{{ transport }}-resource.yml" + + - name: "Remove the destination directory {{ resource_destination_directory }} on error" + file: + path: "{{ resource_destination_directory }}" + state: absent + when: upload_failed + when: create_destination_dir.changed diff --git a/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml b/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml new file mode 100644 index 00000000..4b1b7d75 --- /dev/null +++ b/ansible/roles/resource-data/tasks/unarchive-ssh-resource.yml @@ -0,0 +1,51 @@ +--- +# +# Expected variables +# resources_source_host +# resources_dir +# resource_source_filename +# resource_destination_directory +# Output is upload_failed true/false +# +- name: "Upload resource {{ resources_dir }}/{{ resource_source_filename }} to infrastructure servers over ssh" + block: + - name: Upload ssh private key + copy: + src: "{{ ansible_ssh_private_key_file }}" + dest: /root/.ssh/infra_to_resource.privkey + mode: 0600 + owner: root + group: root + remote_src: no + + - name: Detect if archive is compressed + shell: > + ssh -o StrictHostKeyChecking=no + -i /root/.ssh/infra_to_resource.privkey + {{ resources_source_host }} + 'file "{{ resources_dir }}/{{ resource_source_filename }}"' + | grep "compressed" + register: compressed + + - name: Set tar extract options + set_fact: + tar_extract_options: "{{ '-xzf' if compressed.rc == 0 else '-xf' }}" + + - name: "Unarchive resource {{ resources_dir }}/{{ resource_source_filename }} to {{ resource_destination_directory }} dir on infrastructure servers over ssh" + shell: > + ssh -o StrictHostKeyChecking=no -o BatchMode=yes + -i /root/.ssh/infra_to_resource.privkey + {{ resources_source_host }} + 'cat "{{ resources_dir }}/{{ resource_source_filename }}"' + | tar -C "{{ resource_destination_directory }}" "{{ tar_extract_options }}" - + args: + warn: false + rescue: + - name: Upload failed + set_fact: + upload_failed: true + always: + - name: Remove the ssh private key + file: + path: /root/.ssh/infra_to_resource.privkey + state: absent diff --git a/ansible/roles/resource-data/tasks/upload_resources.yml b/ansible/roles/resource-data/tasks/upload_resources.yml new file mode 100644 index 00000000..571bc7d6 --- /dev/null +++ b/ansible/roles/resource-data/tasks/upload_resources.yml @@ -0,0 +1,15 @@ +--- +- name: Query package facts to check nfs-utils existence + package_facts: + manager: auto + +- name: Set transport fact to nfs or ssh + set_fact: + transport: "{{ 'nfs' if resources_on_nfs and 'nfs-utils' in ansible_facts.packages else 'ssh' }}" + +- name: "Upload resources to infrastructure servers over {{ transport }}" + include_tasks: unarchive-resource.yml + vars: + resource_source_filename: "{{ item.file }}" + resource_destination_directory: "{{ item.destination_dir }}" + loop: "{{ resources_list_of_dicts }}" diff --git a/ansible/site.yml b/ansible/site.yml index 0df534d0..fbf2c389 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -13,7 +13,7 @@ # 3. playbooks here are more or less batch jobs and the ssh authentication # is a precondition, which should be done during configuration of the # installer -- import_playbook: upload_resources.yml +- import_playbook: resources.yml - import_playbook: infrastructure.yml - import_playbook: rancher_kubernetes.yml - import_playbook: application.yml diff --git a/ansible/upload_resources.yml b/ansible/upload_resources.yml deleted file mode 100644 index 68010eb1..00000000 --- a/ansible/upload_resources.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- -- name: Check for presence of auxiliary resources tar file - hosts: resources[0] - tasks: - - name: Store auxiliary resources tar file info into variable - stat: - path: "{{ hostvars[groups.resources.0].resources_dir }}/{{ hostvars[groups.resources.0].aux_resources_filename }}" - register: aux_file_presence - -- name: Check infrastructure server for presence of resources and requirements - hosts: infrastructure - tasks: - - name: Check if nfs-utils is installed - yum: - list: nfs-utils - register: nfs_utils_check - - - name: Check if the resources are already unpacked - stat: - path: "{{ app_data_path }}" - register: resources_data_check - - - name: Check if the auxilliary resources are already unpacked - stat: - path: "{{ aux_data_path }}" - register: aux_resources_data_check - when: aux_data_path is defined and aux_data_path is not none - -- name: Ensure the existence of data directory/ies on infrastructure server - hosts: infrastructure - tasks: - - name: Create data directory - file: - path: "{{ app_data_path }}" - state: directory - - - name: Create auxiliary data directory - file: - path: "{{ aux_data_path }}" - state: directory - when: aux_data_path is defined and aux_data_path is not none - -- name: Upload resources to infrastructure server - hosts: infrastructure - roles: - # use nfs or ssh and unpack resources into data directory/ies - - role: resource-data - vars: - transport: "{{ 'nfs' if resources_on_nfs and (nfs_utils_check.results|selectattr('yumstate', 'match', 'installed')|list|length != 0) else 'ssh' }}" -- cgit 1.2.3-korg