[GATING] Add configuration for Azure3 gating in the fork of chained-ci

This change includes:
* moving submodules of chained-ci-roles and chained-ci-vue as static folders
  to the repo (they were quite old and not updated for some time)
* create azure access artifacts
* add config for azure3 gating pipeline

Issue-ID: INT-2207
Signed-off-by: Marek Szwałkiewicz <marek.szwalkiewicz@external.t-mobile.pl>
Change-Id: Idb475c166d78f10ed4204153ab634110aa9093f6

8 files changed, 395 insertions, 0 deletions

diff --git a/roles/get_artifacts/defaults/main.yml b/roles/get_artifacts/defaults/main.yml
new file mode 100644
index 0000000..112aa4a
--- /dev/null
+++ b/roles/get_artifacts/defaults/main.yml
@@ -0,0 +1,7 @@
+---
+previous_artifacts_folder: "{{ playbook_dir }}/previous_artifacts"
+final_artifacts_folder: "{{  playbook_dir }}/FINAL_ARTIFACT"
+
+job_id_fetch:
+  max_page: 100
+  per_page: 100
diff --git a/roles/get_artifacts/filter_plugins/filters.py b/roles/get_artifacts/filter_plugins/filters.py
new file mode 100644
index 0000000..db38fc6
--- /dev/null
+++ b/roles/get_artifacts/filter_plugins/filters.py
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+sys.path.append(os.path.normpath(os.path.join(os.path.dirname(__file__),'../../')))
+
+from library.filepath import FilterModule
diff --git a/roles/get_artifacts/tasks/binary.yml b/roles/get_artifacts/tasks/binary.yml
new file mode 100644
index 0000000..99ba930
--- /dev/null
+++ b/roles/get_artifacts/tasks/binary.yml
@@ -0,0 +1,244 @@
+---
+##
+# Handle different get_artifacts types
+##
+- name: value change for coherency
+  set_fact:
+    config: >-
+      {{ config|combine({'get_artifacts': [] }) }}
+  when: config.get_artifacts is not defined
+- name: value change for coherency
+  set_fact:
+    config: >-
+      {{ config|combine({'get_artifacts':
+            [{ 'name': config.get_artifacts }] }) }}
+  when: config.get_artifacts is string
+
+- debug:
+    var: config
+    verbosity: 3
+##
+# Prepare a folder for
+##
+
+- name: set previous_artifacts_folder
+  file:
+    path: "{{ item }}"
+    state: directory
+  loop:
+    - "{{ previous_artifacts_folder }}"
+    - "{{ final_artifacts_folder }}"
+
+- name: create dest folders for the jobs artifacts
+  file:
+    path: "{{ previous_artifacts_folder }}/{{ item.name }}"
+    state: directory
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+##
+# Get all artifacts job ids
+##
+- name: loop on get_artifacts
+  include_tasks: get_one_artifact.yml
+  vars:
+    artifact_job_name: "{{ item.name }}"
+    artifact_in_pipeline: "{{ item.in_pipeline | default(true) }}"
+  when: not (item.static_src | default(false))
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ artifact_job_name }}"
+
+- name: download all job artifacts
+  uri:
+    url: >-
+      {{ gitlab.api_url }}/projects/{{ lookup('env', 'CI_PROJECT_ID')
+      }}/jobs/{{ artifact_job_ids[idx] }}/artifacts
+    headers:
+      PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+    status_code: 200
+    dest: >-
+      {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+  when: not (item.static_src | default(false))
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    index_var: idx
+    label: "{{ item.name }}"
+
+- name: download all static artifacts on public projects
+  uri:
+    url: >-
+      {{ config.url }}/raw/{{ config.branch }}/{{
+      config.path | default('') }}/config/artifacts/{{
+      item.name }}.zip?inline=false
+    status_code: 200
+    dest: >-
+      {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+  when: (item.static_src | default(false)) and (config.api is not defined)
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: download all static artifacts using api
+  uri:
+    url: >-
+      {{ config.api }}/repository/files/{{
+      [config.path | default('') , 'config/artifacts'] |
+      filepath(item.name, '.zip')
+      }}/raw?ref={{ config.branch }}
+    headers:
+      PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+    status_code: 200
+    dest: >-
+      {{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip
+  when: (item.static_src | default(false)) and (config.api is defined)
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: unarchive all artifacts
+  unarchive:
+    src: "{{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip"
+    dest: "{{ previous_artifacts_folder }}/{{ item.name }}/"
+    remote_src: "yes"
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: remove all artifacts archives
+  file:
+    path: "{{ previous_artifacts_folder }}/{{ item.name }}/artifacts.zip"
+    state: absent
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: create artifacts folders
+  file:
+    path: "{{ final_artifacts_folder }}/{{ item }}"
+    state: directory
+    recurse: true
+    mode: 0775
+  when: item[-1] == '/'
+  with_items: "{{ vars['.artifacts_root'].paths }}"
+
+- name: copy all files if no filters
+  copy:
+    decrypt: false
+    src: "{{ previous_artifacts_folder }}/{{ item.name }}/"
+    dest: "{{ final_artifacts_folder }}/"
+  when: item.limit_to is not defined or item.limit_to == None
+  loop: "{{ config.get_artifacts }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: copy filtered files if filters
+  include_tasks: limit_to.yml
+  when: item.limit_to is defined
+  loop: "{{ config.get_artifacts }}"
+  vars:
+    job_name: "{{ item.name }}"
+    limit_to: "{{ item.limit_to }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+##
+# get list of files to archive
+##
+- name: get list of files to encrypt
+  find:
+    paths: "{{ final_artifacts_folder }}"
+    recurse: true
+  register: artifacts_files
+
+- name: set file list
+  set_fact:
+    files_list: "{{ artifacts_files.files | map(attribute='path')| list }}"
+
+##
+# If we encode file via ansible vault
+##
+- name: encrypt files
+  shell: >
+    ansible-vault encrypt --vault-password-file {{
+    lookup( 'env', 'VAULT_FILE') }} {{ item }}
+  register: res
+  loop: "{{ files_list }}"
+  failed_when:
+    res.rc == 1 and res.stderr != "ERROR! input is already encrypted"
+  when:
+    config.get_encrypt is defined and (config.get_encrypt | bool)
+
+
+##
+# Add ssh_gateways file if needed
+##
+
+- name: get config step parameters
+  set_fact:
+    config_step: >-
+      {{ gitlab.git_projects[
+           hostvars[inventory_hostname].scenario_steps['config'].project] |
+         combine(hostvars[inventory_hostname].scenario_steps['config']) }}
+
+- name: get ssh gateways config
+  uri:
+    url: >-
+      {{ config_step.api }}/repository/files/{{
+      [config_step.path | default(''), 'config/ssh_gateways'] |
+      filepath(config.ssh_access)
+      }}?ref={{ config_step.branch }}
+    headers:
+      PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+    status_code: 200
+    return_content: yes
+  register: ssh_gw_get
+  when: config.ssh_access is defined
+
+- name: save ssh gateways config
+  copy:
+    content: "{{ ssh_gw_get.json.content | b64decode }}"
+    dest: "{{ final_artifacts_folder }}/vars/ssh_gateways.yml"
+    force: true
+    mode: 0660
+  when: config.ssh_access is defined
+
+##
+# get list of files and folders to archive
+##
+- name: set file list
+  set_fact:
+    arch_files:
+      "{{ (arch_files | default([])) +
+          [ final_artifacts_folder + '/' + item ] }}"
+  loop: "{{ vars['.artifacts_root'].paths }}"
+
+- name: Prepare artifact archive for binary transmission
+  archive:
+    path: "{{ arch_files }}"
+    dest: "{{ playbook_dir }}/artifacts.zip"
+    format: zip
+
+##
+# Set the artifact to send
+##
+- name: "Prepare artifact archive for binary transmission"
+  slurp:
+    src: artifacts.zip
+  register: slurped_artifact
+
+- name: Add artifacts bin if requested
+  set_fact:
+    artifacts_bin: "{{ slurped_artifact.content }}"
+
+##
+# Clean
+##
+- name: delete temporary folders
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "{{ previous_artifacts_folder }}"
+    - "{{ final_artifacts_folder }}"
diff --git a/roles/get_artifacts/tasks/get_one_artifact.yml b/roles/get_artifacts/tasks/get_one_artifact.yml
new file mode 100644
index 0000000..ccbdc48
--- /dev/null
+++ b/roles/get_artifacts/tasks/get_one_artifact.yml
@@ -0,0 +1,49 @@
+---
+##
+# Search for a job id
+#    with name: artifact_job_name
+#    limit to pipeline if artifact_in_pipeline (default: true)
+##
+
+- name: set empty fact for job
+  set_fact:
+    job: {}
+    artifact_in_pipeline: "{{ artifact_in_pipeline | default(true) }}"
+
+- name: get job id in this pipeline
+  when: artifact_in_pipeline | bool
+  block:
+    - name: "Get job successful job ids of the pipeline"
+      uri:
+        url: >-
+          {{ gitlab.api_url }}/projects/{{
+          lookup( 'env', 'CI_PROJECT_ID') }}/pipelines/{{
+          lookup( 'env', 'CI_PIPELINE_ID') }}/jobs?scope[]=success
+        method: GET
+        headers:
+          PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+      register: pipeline_success_jobs
+    - name: get the job id
+      set_fact:
+        job: >-
+          {{ { 'id':
+                  pipeline_success_jobs.json |json_query(
+                    '[?name==`'+ artifact_job_name + ':' +
+                    inventory_hostname +'`].id') | last
+              } }}
+
+- name: fetch the job id corresponding to get_artifact value if not in pipeline
+  include_tasks: job_id_fetch.yml
+  loop: "{{ range(0, job_id_fetch.max_page)| list }}"
+  when: not (artifact_in_pipeline | bool )
+  loop_control:
+    loop_var: page
+
+- name: check we found an artifact job id
+  fail:
+    msg: 'We can not found a correct job id'
+  when: job.id is not defined
+
+- name: get last successful job id
+  set_fact:
+    artifact_job_ids: "{{ (artifact_job_ids|default([])) + [job.id] }}"
diff --git a/roles/get_artifacts/tasks/job_id_fetch.yml b/roles/get_artifacts/tasks/job_id_fetch.yml
new file mode 100644
index 0000000..cab4bcb
--- /dev/null
+++ b/roles/get_artifacts/tasks/job_id_fetch.yml
@@ -0,0 +1,20 @@
+---
+
+- block:
+  - name: "Get successful job ids if artifact fetching"
+    uri:
+      url: >-
+        {{ gitlab.api_url }}/projects/{{ lookup( 'env', 'CI_PROJECT_ID')
+        }}/jobs?scope[]=success&per_page={{ job_id_fetch.per_page
+        }}&page={{ page }}
+      method: GET
+      headers:
+        PRIVATE-TOKEN: "{{ gitlab.private_token }}"
+    register: successful_jobs
+  - name: save successful job
+    set_fact:
+      job: >-
+        {{ successful_jobs.json|
+           selectattr('name', 'equalto', artifact_job_name)| list |
+           first | default({}) }}
+  when: job.id is not defined
diff --git a/roles/get_artifacts/tasks/limit_to.yml b/roles/get_artifacts/tasks/limit_to.yml
new file mode 100644
index 0000000..2e1b782
--- /dev/null
+++ b/roles/get_artifacts/tasks/limit_to.yml
@@ -0,0 +1,20 @@
+---
+
+- debug:
+    var: limit_to
+    verbosity: 3
+- debug:
+    var: job_name
+    verbosity: 3
+- name: copy all files if filters and rename if needed
+  copy:
+    decrypt: false
+    src: "{{ previous_artifacts_folder }}/{{ job_name }}/{{ original }}"
+    dest: "{{ final_artifacts_folder }}/{{ renamed }}"
+  loop: "{{ limit_to }}"
+  vars:
+    original: "{{ file.keys()|first }}"
+    renamed: "{{ file.values()|first }}"
+  loop_control:
+    loop_var: file
+    label: "{{ original }}"
diff --git a/roles/get_artifacts/tasks/main.yml b/roles/get_artifacts/tasks/main.yml
new file mode 100644
index 0000000..605521c
--- /dev/null
+++ b/roles/get_artifacts/tasks/main.yml
@@ -0,0 +1,34 @@
+---
+##
+# Check config is prepared
+##
+- name: check 'step' is set
+  fail:
+    msg: 'Prepare role must be run before'
+  when: config is not defined
+
+
+- name: recover previous artifacts
+  when:
+    config.get_artifacts is defined and
+    config.get_artifacts
+  block:
+    ##
+    # If we get previous artifacts via url
+    ##
+    - name: Add artifacts via source
+      include_tasks: url.yml
+      when:
+        (config.get_bin is not defined or not (config.get_bin | bool))
+        and (config.ssh_access is not defined)
+        and (config.get_artifacts is string)
+
+    ##
+    # If we get previous artifacts via url
+    ##
+    - name: Add artifacts via binary
+      include_tasks: binary.yml
+      when:
+        (config.get_bin is defined and (config.get_bin | bool))
+        or (config.ssh_access is defined)
+        or (config.get_artifacts is not string)
diff --git a/roles/get_artifacts/tasks/url.yml b/roles/get_artifacts/tasks/url.yml
new file mode 100644
index 0000000..a2b5a91
--- /dev/null
+++ b/roles/get_artifacts/tasks/url.yml
@@ -0,0 +1,13 @@
+---
+
+- name: get_artifacts with just one value
+  include_tasks: get_one_artifact.yml
+  vars:
+    artifact_job_name: "{{ config.get_artifacts }}"
+
+- name: get the url of the artifact
+  set_fact:
+    artifacts_src: >-
+      {{ gitlab.api_url }}/projects/{{
+      lookup( 'env', 'CI_PROJECT_ID') }}/jobs/{{
+      artifact_job_ids[0] }}/artifacts