Release oom-automatic-installation

Issue-ID: INT-2150
Signed-off-by: Michal Jagiello <michal.jagiello@t-mobile.pl>
Change-Id: I8992f93e2b79f17c90d8f3c350891e8428ef3dcb

8 files changed, 481 insertions, 0 deletions

diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 0000000..abebcdd
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,76 @@
+# chained-ci-tools
+
+Library to unify the usage of chained-ci
+
+How to run
+----
+
+To prepare the environment just run:
+```./<chained-ci-tools_folder>/chained-ci-init.sh [-a] [-i inventory]```
+
+This will prepare depending on your artifacts and env vars:
+
+- The vault key file
+- Get the artifacts that came from chained-ci
+- Set the ssh key and the ssh config
+
+Options are:
+- ```-a```: Read the remote artifact
+- ```-i inventory```: Set the inventory file for ssh config
+
+For security purpose, the environment should ALWAYS be clean! the proposed
+script underneath will:
+- Remove the vault key file
+- Remove the ssh config file with id_rsa key files
+- Vault ALL the artifact files of the current job. To add an exception, and do
+  not vault a file, or a folder, you can set the NOVAULT_LIST parameter filled
+  with paths separated by a carriage return or a space, like this:
+  ```
+  NOVAULT_LIST="""folder1/file2
+  folder2/file2
+  folder3/"""
+  ```
+  or
+  ```
+  NOVAULT_LIST="folder1/file2 folder2/file2 folder3/"
+  ```
+  Please note the '/' at the end of the folder; it will work without but you may
+  also filter all names starting with "folder3"
+
+
+to use the clean script of the environment, just run:
+```
+./<chained-ci-tools_folder>/clean.sh
+```
+
+
+Use it as a submodule
+----------
+
+```
+git submodule add https://gitlab.com/Orange-OpenSource/lfn/ci_cd/chained-ci-tools.git scripts/chained-ci-tools
+```
+
+If you use the CI, don't forget to add the following parameter in ```.gitlab-ci.yml```
+```
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+```
+
+
+Chained-ci-tools in gitlab-ci.yml
+--------
+
+In your ```.gitlab-ci.yml```, you can add:
+```
+.chained_ci_tools: &chained_ci_tools
+  before_script:
+    - ./scripts/chained-ci-tools/chained-ci-init.sh -a -i inventory
+  after_script:
+    - ./scripts/chained-ci-tools/clean.sh
+```
+
+and add this block when you need to run it
+```
+<<: *chained_ci_tools
+```
diff --git a/scripts/chained-ci-init.sh b/scripts/chained-ci-init.sh
new file mode 100755
index 0000000..1ea78e2
--- /dev/null
+++ b/scripts/chained-ci-init.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+export RUN_SCRIPT=${BASH_SOURCE[0]}
+
+if [ -r $1 ]; then
+  echo """
+<!> DEPRECATION <!>
+<!> You are using a deprecated call to this script.
+<!> Please use the following options:
+<!>   -i inventory : to set the inventory path to generate the ssh config file
+<!>   -a           : to read the remote artifact
+"""
+  DEPRECATED_WAY="True"
+  INVENTORY=$1
+  REMOTE_ARTIFACT="True"
+else
+  while getopts ai: option
+  do
+    case "${option}"
+    in
+      a) REMOTE_ARTIFACT="True";; # Read the remote artifact
+      i) INVENTORY=${OPTARG};;    # Set the inventory file for ssh config
+    esac
+  done
+fi
+
+export TOOLS_FOLDER=$(dirname $(readlink -f ${RUN_SCRIPT}))
+export ROOT_FOLDER=${PWD}
+. ${TOOLS_FOLDER}/rc.sh
+trap submit_bug_report ERR
+
+##############################################
+step_banner "Tasked trigger infos"
+##############################################
+echo "POD: ${pod}"
+echo "Pipeline triggered by: ${source_job_name}"
+
+##############################################
+step_banner "Prepare environment"
+##############################################
+
+# Set Vault password
+VAULT_OPT=''
+if [ -n "${ANSIBLE_VAULT_PASSWORD}" ]; then
+  step_line "ansible vault password file"
+  echo ${ANSIBLE_VAULT_PASSWORD} > ${ROOT_FOLDER}/.vault
+  export VAULT_OPT="--vault-password-file ${ROOT_FOLDER}/.vault"
+else
+  step_line no vault password provided
+fi
+
+##############################################
+step_banner "Get artifacts"
+##############################################
+if [ "${CI_PIPELINE_SOURCE}" == "trigger" ] && [ "${REMOTE_ARTIFACT}" == "True" ]; then
+  if [ -n "${artifacts_src}" ] || [ -n "${artifacts_bin}" ]; then
+    if [ -n "${artifacts_src}" ]; then
+      step_line "getting artifact from source url"
+      step_line "(your may need to set PRIVATE_TOKEN argument to access non public artifact)"
+      curl -L -s -H "PRIVATE-TOKEN: ${PRIVATE_TOKEN}" -o "${ROOT_FOLDER}/artifacts.zip" "${artifacts_src}"
+    elif [ -n "${artifacts_bin}" ]; then
+      step_line "getting artifact from its binary content"
+      echo "${artifacts_bin}" | base64 -d > ${ROOT_FOLDER}/artifacts.zip
+    fi
+    step_line "unzip artifacts"
+    unzip -o ${ROOT_FOLDER}/artifacts.zip -d ${ROOT_FOLDER}
+    rm ${ROOT_FOLDER}/artifacts.zip
+  else
+    step_line "No artifact provided"
+    exit -1
+  fi
+else
+  step_line "Pipeline not triggered (\$CI_PIPELINE_SOURCE=${CI_PIPELINE_SOURCE})"
+  step_line "or remote artifact option '-a' not set"
+fi
+
+##############################################
+step_banner "Set SSH config"
+##############################################
+if [ -e ${ROOT_FOLDER}/vars/vaulted_ssh_credentials.yml ]; then
+  if [ -z "${INVENTORY}" ]; then
+    error_line "No Inventory provided (-i option)"
+    exit -1
+  else
+    check_ci_var ANSIBLE_VAULT_PASSWORD
+    check_ci_var INVENTORY
+    step_line Generate SSH config
+    ansible-playbook ${ansible_verbose} -i ${INVENTORY} ${VAULT_OPT} ${TOOLS_FOLDER}/prepare_ssh.yml
+    export SSH_OPT="-F ${ROOT_FOLDER}/ssh_config"
+    export ANSIBLE_SSH_ARGS="-C -o ControlMaster=auto -o ControlPersist=60s ${SSH_OPT}"
+    if [ "${DEPRECATED_WAY}" == "True" ]; then
+      step_line Add symlink to support DEPRECATED calls of this script
+      ln -s ${ROOT_FOLDER}/ssh_config ${ROOT_FOLDER}/config
+    fi
+  fi
+else
+  step_line "no ssh creds"
+fi
+
+##############################################
+step_banner "End of preparation"
+##############################################
diff --git a/scripts/clean.sh b/scripts/clean.sh
new file mode 100755
index 0000000..7f255fc
--- /dev/null
+++ b/scripts/clean.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env sh
+
+export TOOLS_FOLDER=$(dirname $(readlink -f ${0}))
+export ROOT_FOLDER=${PWD}
+. ${TOOLS_FOLDER}/rc.sh
+
+###############################################################
+step_banner Artifact ciphering
+###############################################################
+
+# Function to check a file is in a list
+file_in_list (){
+  LIST=$(echo $1|tr '\\n' ' ') #if we send it with CR separator
+  FILE=$2
+  for FILTER in ${LIST}; do
+    if $(echo ${FILE}| grep "^${FILTER}" 2>&1 >/dev/null); then
+      return 0
+    fi
+  done
+  return 1
+}
+
+if [ -e ${ROOT_FOLDER}/.vault ]; then
+  #Ensure we have a NOVAULT_LIST
+  NOVAULT_LIST="fake/file ${NOVAULT_LIST}"
+  #Get artifacts paths
+  INV_PATHS=$(cat .gitlab-ci.yml | yq --arg job ${CI_JOB_NAME} -r '.[$job].artifacts.paths[]')
+  #Read paths
+  for INV_PATH in ${INV_PATHS}; do
+    if [ -e ${INV_PATH} ]; then
+      #If the artifact is a directory, reads files in it
+      if [ -d ${INV_PATH} ]; then
+        FILES=$(find ${INV_PATH} -type f)
+      else
+        FILES=${INV_PATH}
+      fi
+      # For each file, vault or not
+      for FILE in ${FILES}; do
+        if $(file_in_list "${NOVAULT_LIST}" ${FILE}); then
+          echo "${FILE}: Not vaulting"
+        else
+          if $(head -n1 ${FILE} |grep "^\$ANSIBLE_VAULT;" > /dev/null); then
+            echo "${FILE}: Already vaulted"
+          else
+            echo "${FILE}: Vaulting"
+            ansible-vault encrypt --vault-password-file ${ROOT_FOLDER}/.vault ${FILE}
+          fi
+        fi
+      done
+    fi
+  done
+fi
+
+###############################################################
+step_banner Cleaning all files
+###############################################################
+if [ -e ${ROOT_FOLDER}/.vault ]; then
+  step_line remove vault file
+  rm ${ROOT_FOLDER}/.vault
+fi
+if [ -e ${ROOT_FOLDER}/id_rsa ]; then
+  step_line remove ssh certs
+  rm  ${ROOT_FOLDER}/id_rsa
+fi
+if [ -e ${ROOT_FOLDER}/id_rsa.pub ]; then
+  step_line remove pub ssh certs
+  rm  ${ROOT_FOLDER}/id_rsa.pub
+fi
+if [ -e ${ROOT_FOLDER}/ssh_config ]; then
+  step_line remove ssh config
+  rm  ${ROOT_FOLDER}/ssh_config
+fi
+if [ -e ${ROOT_FOLDER}/vars/openstack_openrc ]; then
+  step_line remove openstack admin rc
+  rm  ${ROOT_FOLDER}/vars/openstack_openrc
+fi
diff --git a/scripts/prepare_ssh.yml b/scripts/prepare_ssh.yml
new file mode 100644
index 0000000..cc09d05
--- /dev/null
+++ b/scripts/prepare_ssh.yml
@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: "no"
+  roles:
+    - ssh_prepare
diff --git a/scripts/rc.sh b/scripts/rc.sh
new file mode 100644
index 0000000..4ab435d
--- /dev/null
+++ b/scripts/rc.sh
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 Orange and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+##
+# Bug report function
+##
+submit_bug_report() {
+  SHELL_VERBOSE=${SHELL_VERBOSE:-}
+  local lc="$BASH_COMMAND" rc=$?
+  echo ""
+  echo "---------------------------------------------------------------------"
+  step_line "Crash on command"
+  echo
+  echo $lc
+  echo
+  step_line "Exit code"
+  echo
+  echo $rc
+  if [ ! -z ${SHELL_VERBOSE} ]; then
+    echo
+    step_line "Environment variables"
+    echo
+    env | grep -v artifacts_bin \
+        | sort \
+        | sed 's/^/export /' \
+        | sed 's/PASSWORD=.*/PASSWORD=***HIDDEN***/' \
+        | sed 's/artifacts_bin=.*/artifacts_bin=***HIDDEN***/'
+  fi
+  echo "---------------------------------------------------------------------"
+  step_banner "Clean"
+  ${TOOLS_FOLDER}/clean.sh
+  echo "---------------------------------------------------------------------"
+}
+
+
+##
+# Pretty print
+##
+step_banner() {
+  echo ""
+  echo "====================================================================="
+  echo "${RUN_SCRIPT}"
+  date
+  echo "$*"
+  echo "====================================================================="
+  echo ""
+}
+
+step_line() {
+  echo ">>> ${*}"
+}
+
+error_line() {
+  echo "!!! ${*}"
+}
+
+##
+# Test A CI var is required
+##
+check_ci_var() {
+  var_name=$1
+  if [ -z "${!var_name}" ]; then
+    error_line
+    error_line "Variable \$${var_name} must be defined"
+    error_line "Please set it in your gitlab project (Settings / CI-CD / variables page)"
+    error_line
+    exit
+  fi
+}
+
+##
+# Warn if run as root
+##
+no_root_needed() {
+  step_line "Check if we are root"
+  if [[ $(whoami) == "root" ]]; then
+      echo "WARNING: This script should not be run as root!"
+      echo "Elevated privileges are aquired automatically when necessary"
+      echo "Waiting 10s to give you a chance to stop the script (Ctrl-C)"
+      for x in $(seq 10 -1 1); do echo -n "$x..."; sleep 1; done
+  fi
+}
+
+##
+# Ensure root folder is not world readable
+##
+ansible_prepare(){
+  step_line "Set local folder not writable to others"
+  chmod 600 ${ROOT_FOLDER}
+}
+
+##
+# SSH Options
+##
+ssh_opt(){
+  SSH_OPT=''
+  if [ -e ${ROOT_FOLDER}/vars/vaulted_ssh_credentials.yml ]; then
+    SSH_OPT="${SSH_OPT} -F ${ROOT_FOLDER}/ssh_config"
+  fi
+  echo ${SSH_OPT}
+}
+
+##
+# Vault Options
+##
+vault_opt(){
+  VAULT_OPT=''
+  if [ -n "${ANSIBLE_VAULT_PASSWORD}" ]; then
+    VAULT_OPT="--vault-password-file ${ROOT_FOLDER}/.vault"
+  fi
+  echo ${VAULT_OPT}
+}
+
+##
+# Get Ansible SSH Options
+##
+ansible_ssh_opt(){
+  ANSIBLE_SSH_ARGS="-C -o ControlMaster=auto -o ControlPersist=60s"
+  if [ -n "${ANSIBLE_VAULT_PASSWORD}" ]; then
+    ANSIBLE_SSH_ARGS="${ANSIBLE_SSH_ARGS} $(ssh_opt)"
+  fi
+  echo ${ANSIBLE_SSH_ARGS}
+}
+
+##
+# Cat file that may be vaulted
+##
+cat_file(){
+  FILE=$1
+  if [ -e ${ROOT_FOLDER}/.vault ] \
+     && $(grep '^\$ANSIBLE_VAULT;1\..;AES256' ${FILE} > /dev/null); then
+    ansible-vault view --vault-password-file=${ROOT_FOLDER}/.vault ${FILE}
+  else
+    cat ${FILE}
+  fi
+}
diff --git a/scripts/ssh_prepare/defaults/main.yml b/scripts/ssh_prepare/defaults/main.yml
new file mode 100644
index 0000000..f074f01
--- /dev/null
+++ b/scripts/ssh_prepare/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+# variable needed to access jumphost
+ssh_id_rsa: "{{ vault_ssh_id_rsa }}"
+ssh_id_rsa_pub: "{{ vault_ssh_id_rsa_pub }}"
diff --git a/scripts/ssh_prepare/tasks/main.yml b/scripts/ssh_prepare/tasks/main.yml
new file mode 100644
index 0000000..e47ab11
--- /dev/null
+++ b/scripts/ssh_prepare/tasks/main.yml
@@ -0,0 +1,40 @@
+---
+- set_fact:
+    base_dir: "{{ lookup('env', 'ROOT_FOLDER') | default(playbook_dir, true) }}"
+
+- name: check if vaulted ssh credentials exists
+  stat:
+    path: "{{ base_dir }}/vars/vaulted_ssh_credentials.yml"
+  register: creds_stat
+
+- name: include vaulted ssh credentials
+  include_vars: "{{ base_dir }}/vars/vaulted_ssh_credentials.yml"
+  when: creds_stat.stat.exists
+
+- name: check if vaulted ssh_gateways file exists
+  stat:
+    path: "{{ base_dir }}/vars/ssh_gateways.yml"
+  register: gw_stat
+
+- name: include vaulted ssh gateways
+  include_vars: "{{ base_dir }}/vars/ssh_gateways.yml"
+  when: gw_stat.stat.exists
+
+- name: create id_rsa file
+  copy:
+    dest: "{{ base_dir }}/id_rsa"
+    content: "{{ ssh_id_rsa }}"
+    mode: 0600
+  when: creds_stat.stat.exists
+
+- name: create id_rsa.pub file
+  copy:
+    dest: "{{ base_dir }}/id_rsa.pub"
+    content: "{{ ssh_id_rsa_pub }}"
+    mode: 0600
+  when: creds_stat.stat.exists
+
+- name: generate ssh config
+  template:
+    src: config.j2
+    dest: "{{ base_dir }}/ssh_config"
diff --git a/scripts/ssh_prepare/templates/config.j2 b/scripts/ssh_prepare/templates/config.j2
new file mode 100644
index 0000000..375efd7
--- /dev/null
+++ b/scripts/ssh_prepare/templates/config.j2
@@ -0,0 +1,33 @@
+Host *
+{% if creds_stat.stat.exists %}
+  IdentityFile {{ base_dir }}/id_rsa
+{% endif %}
+  UserKnownHostsFile=/dev/null
+  StrictHostKeyChecking=no
+
+{% if gw_stat.stat.exists %}
+{% for gw in ssh_gateways | default([]) %}
+host {{ gw.name }}
+  Hostname {{ gw.public_fqdn | default(gw.ansible_host) }}
+  User {{ gw.ansible_user }}
+{% if gw.ansible_port is defined %}
+  Port {{ gw.ansible_port }}
+{% endif %}
+{% if gw.proxy_command is defined %}
+  ProxyCommand  {{ gw.proxy_command }}
+{% endif %}
+
+{% endfor %}
+{% endif %}
+
+{% for node in groups.all %}
+{% if hostvars[node].ansible_host is defined %}
+host {{ node }} {{ hostvars[node].public_fqdn | default('') }} {{ hostvars[node].ansible_host }}
+  Hostname {{ hostvars[node].public_fqdn | default(hostvars[node].ansible_host) }}
+  User {{ hostvars[node].ansible_user }}
+{% if gw_stat.stat.exists %}
+  ProxyCommand ssh -F {{ base_dir }}/ssh_config -W %h:%p {{ ssh_gateways[0].name }}
+{% endif %}
+{% endif %}
+
+{% endfor %}