aboutsummaryrefslogtreecommitdiffstats
path: root/roles/get_artifacts
diff options
context:
space:
mode:
authorMarek Szwałkiewicz <marek.szwalkiewicz@external.t-mobile.pl>2023-03-01 12:27:28 +0100
committerMarek Szwałkiewicz <marek.szwalkiewicz@external.t-mobile.pl>2023-03-03 13:46:02 +0100
commit70fa03898ee412e30b6b87cf961004bf16ccaef4 (patch)
tree10cca3196bd5db69ee643316365a00f1276dba04 /roles/get_artifacts
parent0399d9842c2a5670e4ee21d45343d2ac168eee2d (diff)
[GATING] Add configuration for Azure3 gating in the fork of chained-ci
This change includes: * moving submodules of chained-ci-roles and chained-ci-vue as static folders to the repo (they were quite old and not updated for some time) * create azure access artifacts * add config for azure3 gating pipeline Issue-ID: INT-2207 Signed-off-by: Marek Szwałkiewicz <marek.szwalkiewicz@external.t-mobile.pl> Change-Id: Idb475c166d78f10ed4204153ab634110aa9093f6
Diffstat (limited to 'roles/get_artifacts')
-rw-r--r--roles/get_artifacts/defaults/main.yml7
-rw-r--r--roles/get_artifacts/filter_plugins/filters.py8
-rw-r--r--roles/get_artifacts/tasks/binary.yml244
-rw-r--r--roles/get_artifacts/tasks/get_one_artifact.yml49
-rw-r--r--roles/get_artifacts/tasks/job_id_fetch.yml20
-rw-r--r--roles/get_artifacts/tasks/limit_to.yml20
-rw-r--r--roles/get_artifacts/tasks/main.yml34
-rw-r--r--roles/get_artifacts/tasks/url.yml13
8 files changed, 395 insertions, 0 deletions
diff --git a/roles/get_artifacts/defaults/main.yml b/roles/get_artifacts/defaults/main.yml
new file mode 100644
index 0000000..112aa4a
--- /dev/null
+++ b/roles/get_artifacts/defaults/main.yml
@@ -0,0 +1,7 @@
+---
+previous_artifacts_folder: "{{ playbook_dir }}/previous_artifacts"
+final_artifacts_folder: "{{  playbook_dir }}/FINAL_ARTIFACT"
+
+job_id_fetch:
+ max_page: 100
+ per_page: 100
diff --git a/roles/get_artifacts/filter_plugins/filters.py b/roles/get_artifacts/filter_plugins/filters.py
new file mode 100644
index 0000000..db38fc6
--- /dev/null
+++ b/roles/get_artifacts/filter_plugins/filters.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__),'../../')))
+
+from library.filepath import FilterModule
diff --git a/roles/get_artifacts/tasks/binary.yml b/roles/get_artifacts/tasks/binary.yml
new file mode 100644
index 0000000..99ba930
--- /dev/null
+++ b/roles/get_artifacts/tasks/binary.yml
@@ -0,0 +1,244 @@
+---
+##
+# Handle different get_artifacts types
+##
+- name: value change for coherency
+ set_fact:
+ config: >-
+ {{ config|combine({'get_artifacts': [] }) }}
+ when: config.get_artifacts is not defined
+- name: value change for coherency
+ set_fact:
+ config: >-
+ {{ config|combine({'get_artifacts':
+ [{ 'name': config.get_artifacts }] }) }}
+ when: config.get_artifacts is string
+
+- debug:
+ var: config
+ verbosity: 3
+##
+# Prepare a folder for
+##
+
+- name: set previous_artifacts_folder
+ file:
+ path: "{{ item }}"
+ state: directory
+ loop:
+ - "{{ previous_artifacts_folder }}"
+ - "{{ final_artifacts_folder }}"
+
+- name: create dest folders for the jobs artifacts
+ file:
+ path: "{{ previous_artifacts_folder }}/{{ item.name }}"
+ state: directory
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+##
+# Get all artifacts job ids
+##
+- name: loop on get_artifacts
+ include_tasks: get_one_artifact.yml
+ vars:
+ artifact_job_name: "{{ item.name }}"
+ artifact_in_pipeline: "{{ item.in_pipeline | default(true) }}"
+ when: not (item.static_src | default(false))
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ artifact_job_name }}"
+
+- name: download all job artifacts
+ uri:
+ url: >-
+ {{ gitlab.api_url }}/projects/{{ lookup('env', 'CI_PROJECT_ID')
+ }}/jobs/{{ artifact_job_ids[idx] }}/artifacts
+ headers:
+ PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+ status_code: 200
+ dest: >-
+ {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+ when: not (item.static_src | default(false))
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ index_var: idx
+ label: "{{ item.name }}"
+
+- name: download all static artifacts on public projects
+ uri:
+ url: >-
+ {{ config.url }}/raw/{{ config.branch }}/{{
+ config.path | default('') }}/config/artifacts/{{
+ item.name }}.zip?inline=false
+ status_code: 200
+ dest: >-
+ {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+ when: (item.static_src | default(false)) and (config.api is not defined)
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+- name: download all static artifacts using api
+ uri:
+ url: >-
+ {{ config.api }}/repository/files/{{
+ [config.path | default('') , 'config/artifacts'] |
+ filepath(item.name, '.zip')
+ }}/raw?ref={{ config.branch }}
+ headers:
+ PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+ status_code: 200
+ dest: >-
+ {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+ when: (item.static_src | default(false)) and (config.api is defined)
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+- name: unarchive all artifacts
+ unarchive:
+ src: "{{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip"
+ dest: "{{ previous_artifacts_folder }}/{{ item.name }}/"
+ remote_src: "yes"
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+- name: remove all artifacts archives
+ file:
+ path: "{{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip"
+ state: absent
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+- name: create artifacts folders
+ file:
+ path: "{{ final_artifacts_folder }}/{{ item }}"
+ state: directory
+ recurse: true
+ mode: 0775
+ when: item[-1] == '/'
+ with_items: "{{ vars['.artifacts_root'].paths }}"
+
+- name: copy all files if no filters
+ copy:
+ decrypt: false
+ src: "{{ previous_artifacts_folder }}/{{ item.name }}/"
+ dest: "{{ final_artifacts_folder }}/"
+ when: item.limit_to is not defined or item.limit_to == None
+ loop: "{{ config.get_artifacts }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+- name: copy filtered files if filters
+ include_tasks: limit_to.yml
+ when: item.limit_to is defined
+ loop: "{{ config.get_artifacts }}"
+ vars:
+ job_name: "{{ item.name }}"
+ limit_to: "{{ item.limit_to }}"
+ loop_control:
+ label: "{{ item.name }}"
+
+##
+# get list of files to archive
+##
+- name: get list of files to encrypt
+ find:
+ paths: "{{ final_artifacts_folder }}"
+ recurse: true
+ register: artifacts_files
+
+- name: set file list
+ set_fact:
+ files_list: "{{ artifacts_files.files | map(attribute='path')| list }}"
+
+##
+# If we encode file via ansible vault
+##
+- name: encrypt files
+ shell: >
+ ansible-vault encrypt --vault-password-file {{
+ lookup( 'env', 'VAULT_FILE') }} {{ item }}
+ register: res
+ loop: "{{ files_list }}"
+ failed_when:
+ res.rc == 1 and res.stderr != "ERROR! input is already encrypted"
+ when:
+ config.get_encrypt is defined and (config.get_encrypt | bool)
+
+
+##
+# Add ssh_gateways file if needed
+##
+
+- name: get config step parameters
+ set_fact:
+ config_step: >-
+ {{ gitlab.git_projects[
+ hostvars[inventory_hostname].scenario_steps['config'].project] |
+ combine(hostvars[inventory_hostname].scenario_steps['config']) }}
+
+- name: get ssh gateways config
+ uri:
+ url: >-
+ {{ config_step.api }}/repository/files/{{
+ [config_step.path | default(''), 'config/ssh_gateways'] |
+ filepath(config.ssh_access)
+ }}?ref={{ config_step.branch }}
+ headers:
+ PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+ status_code: 200
+ return_content: yes
+ register: ssh_gw_get
+ when: config.ssh_access is defined
+
+- name: save ssh gateways config
+ copy:
+ content: "{{ ssh_gw_get.json.content | b64decode }}"
+ dest: "{{ final_artifacts_folder }}/vars/ssh_gateways.yml"
+ force: true
+ mode: 0660
+ when: config.ssh_access is defined
+
+##
+# get list of files and folders to archive
+##
+- name: set file list
+ set_fact:
+ arch_files:
+ "{{ (arch_files | default([])) +
+ [ final_artifacts_folder + '/' + item ] }}"
+ loop: "{{ vars['.artifacts_root'].paths }}"
+
+- name: Prepare artifact archive for binary transmission
+ archive:
+ path: "{{ arch_files }}"
+ dest: "{{ playbook_dir }}/artifacts.zip"
+ format: zip
+
+##
+# Set the artifact to send
+##
+- name: "Prepare artifact archive for binary transmission"
+ slurp:
+ src: artifacts.zip
+ register: slurped_artifact
+
+- name: Add artifacts bin if requested
+ set_fact:
+ artifacts_bin: "{{ slurped_artifact.content }}"
+
+##
+# Clean
+##
+- name: delete temporary folders
+ file:
+ path: "{{ item }}"
+ state: absent
+ loop:
+ - "{{ previous_artifacts_folder }}"
+ - "{{ final_artifacts_folder }}"
diff --git a/roles/get_artifacts/tasks/get_one_artifact.yml b/roles/get_artifacts/tasks/get_one_artifact.yml
new file mode 100644
index 0000000..ccbdc48
--- /dev/null
+++ b/roles/get_artifacts/tasks/get_one_artifact.yml
@@ -0,0 +1,49 @@
+---
+##
+# Search for a job id
+# with name: artifact_job_name
+# limit to pipeline if artifact_in_pipeline (default: true)
+##
+
+- name: set empty fact for job
+ set_fact:
+ job: {}
+ artifact_in_pipeline: "{{ artifact_in_pipeline | default(true) }}"
+
+- name: get job id in this pipeline
+ when: artifact_in_pipeline | bool
+ block:
+ - name: "Get job successful job ids of the pipeline"
+ uri:
+ url: >-
+ {{ gitlab.api_url }}/projects/{{
+ lookup( 'env', 'CI_PROJECT_ID') }}/pipelines/{{
+ lookup( 'env', 'CI_PIPELINE_ID') }}/jobs?scope[]=success
+ method: GET
+ headers:
+ PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+ register: pipeline_success_jobs
+ - name: get the job id
+ set_fact:
+ job: >-
+ {{ { 'id':
+ pipeline_success_jobs.json |json_query(
+ '[?name==`'+ artifact_job_name + ':' +
+ inventory_hostname +'`].id') | last
+ } }}
+
+- name: fetch the job id corresponding to get_artifact value if not in pipeline
+ include_tasks: job_id_fetch.yml
+ loop: "{{ range(0, job_id_fetch.max_page)| list }}"
+ when: not (artifact_in_pipeline | bool )
+ loop_control:
+ loop_var: page
+
+- name: check we found an artifact job id
+ fail:
+ msg: 'We can not found a correct job id'
+ when: job.id is not defined
+
+- name: get last successful job id
+ set_fact:
+ artifact_job_ids: "{{ (artifact_job_ids|default([])) + [job.id] }}"
diff --git a/roles/get_artifacts/tasks/job_id_fetch.yml b/roles/get_artifacts/tasks/job_id_fetch.yml
new file mode 100644
index 0000000..cab4bcb
--- /dev/null
+++ b/roles/get_artifacts/tasks/job_id_fetch.yml
@@ -0,0 +1,20 @@
+---
+
+- block:
+ - name: "Get successful job ids if artifact fetching"
+ uri:
+ url: >-
+ {{ gitlab.api_url }}/projects/{{ lookup( 'env', 'CI_PROJECT_ID')
+ }}/jobs?scope[]=success&per_page={{ job_id_fetch.per_page
+ }}&page={{ page }}
+ method: GET
+ headers:
+ PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+ register: successful_jobs
+ - name: save successful job
+ set_fact:
+ job: >-
+ {{ successful_jobs.json|
+ selectattr('name', 'equalto', artifact_job_name)| list |
+ first | default({}) }}
+ when: job.id is not defined
diff --git a/roles/get_artifacts/tasks/limit_to.yml b/roles/get_artifacts/tasks/limit_to.yml
new file mode 100644
index 0000000..2e1b782
--- /dev/null
+++ b/roles/get_artifacts/tasks/limit_to.yml
@@ -0,0 +1,20 @@
+---
+
+- debug:
+ var: limit_to
+ verbosity: 3
+- debug:
+ var: job_name
+ verbosity: 3
+- name: copy all files if filters and rename if needed
+ copy:
+ decrypt: false
+ src: "{{ previous_artifacts_folder }}/{{ job_name }}/{{ original }}"
+ dest: "{{ final_artifacts_folder }}/{{ renamed }}"
+ loop: "{{ limit_to }}"
+ vars:
+ original: "{{ file.keys()|first }}"
+ renamed: "{{ file.values()|first }}"
+ loop_control:
+ loop_var: file
+ label: "{{ original }}"
diff --git a/roles/get_artifacts/tasks/main.yml b/roles/get_artifacts/tasks/main.yml
new file mode 100644
index 0000000..605521c
--- /dev/null
+++ b/roles/get_artifacts/tasks/main.yml
@@ -0,0 +1,34 @@
+---
+##
+# Check config is prepared
+##
+- name: check 'step' is set
+ fail:
+ msg: 'Prepare role must be run before'
+ when: config is not defined
+
+
+- name: recover previous artifacts
+ when:
+ config.get_artifacts is defined and
+ config.get_artifacts
+ block:
+ ##
+ # If we get previous artifacts via url
+ ##
+ - name: Add artifacts via source
+ include_tasks: url.yml
+ when:
+ (config.get_bin is not defined or not (config.get_bin | bool))
+ and (config.ssh_access is not defined)
+ and (config.get_artifacts is string)
+
+ ##
+ # If we get previous artifacts via url
+ ##
+ - name: Add artifacts via binary
+ include_tasks: binary.yml
+ when:
+ (config.get_bin is defined and (config.get_bin | bool))
+ or (config.ssh_access is defined)
+ or (config.get_artifacts is not string)
diff --git a/roles/get_artifacts/tasks/url.yml b/roles/get_artifacts/tasks/url.yml
new file mode 100644
index 0000000..a2b5a91
--- /dev/null
+++ b/roles/get_artifacts/tasks/url.yml
@@ -0,0 +1,13 @@
+---
+
+- name: get_artifacts with just one value
+ include_tasks: get_one_artifact.yml
+ vars:
+ artifact_job_name: "{{ config.get_artifacts }}"
+
+- name: get the url of the artifact
+ set_fact:
+ artifacts_src: >-
+ {{ gitlab.api_url }}/projects/{{
+ lookup( 'env', 'CI_PROJECT_ID') }}/jobs/{{
+ artifact_job_ids[0] }}/artifacts