diff options
Diffstat (limited to 'ansible')
38 files changed, 532 insertions, 386 deletions
diff --git a/ansible/application.yml b/ansible/application.yml index bbac7e5c..02c654f2 100644 --- a/ansible/application.yml +++ b/ansible/application.yml @@ -2,22 +2,16 @@ - name: Setup nfs server hosts: nfs-server roles: - - {role: nfs, when: groups.kubernetes | length > 1 } + - role: nfs + when: groups.kubernetes | length > 1 - name: Setup nfs mounts hosts: kubernetes:!nfs-server roles: - - {role: nfs, when: groups.kubernetes | length > 1 } + - role: nfs + when: groups.kubernetes | length > 1 - name: Install Helm application {{ app_name }} into offline Kubernetes cluster hosts: infrastructure roles: - - role: application-install - vars: - phase: pre-install - - role: application-install - vars: - phase: install - - role: application-install - vars: - phase: post-install + - application diff --git a/ansible/docker/run_chroot.sh b/ansible/docker/run_chroot.sh index 3359fdcd..8ae9c188 100755 --- a/ansible/docker/run_chroot.sh +++ b/ansible/docker/run_chroot.sh @@ -22,7 +22,6 @@ set -e CMD=$(basename "$0") -UMOUNT_TIMEOUT=120 # 2mins # @@ -106,31 +105,18 @@ EXAMPLE: " } -# arg: <directory> -is_mounted() -{ - mountpoint=$(echo "$1" | sed 's#//*#/#g') - - LANG=C mount | grep -q "^[^[:space:]]\+[[:space:]]\+on[[:space:]]\+${mountpoint}[[:space:]]\+type[[:space:]]\+" -} - # layers are right to left! First is on the right, top/last is on the left do_overlay_mount() { - if [ -d "$overlay" ] && is_mounted "$overlay" ; then - echo ERROR: "The overlay directory is already mounted: $overlay" >&2 - echo ERROR: "Fix the issue - cannot proceed" >&2 - exit 1 - fi - # prepare dirs - rm -rf "$overlay" "$upperdir" "$workdir" +mkdir -p $ovtempdir +mount -t tmpfs -o mode=0755 tmpfs $ovtempdir mkdir -p "$overlay" mkdir -p "$upperdir" mkdir -p "$workdir" # finally overlay mount - if ! mount -t overlay --make-rprivate \ + if ! mount -t overlay \ -o lowerdir="$lowerdir",upperdir="$upperdir",workdir="$workdir" \ overlay "$overlay" ; then @@ -147,40 +133,16 @@ do_overlay_mount() return 0 } -cleanup() -{ - case "$OVERLAY_MOUNT" in - yes) - echo INFO: "Umounting overlay..." >&2 - if ! umount_retry "$CHROOT_DIR" ; then - echo ERROR: "Cannot umount chroot: $CHROOT_DIR" >&2 - return 1 - fi - - ;; - no) - echo INFO: "No overlay to umount" >&2 - ;; - esac - - if ! is_mounted "$overlay" ; then - echo INFO: "Deleting of temp directories..." >&2 - rm -rf "$overlay" "$upperdir" "$workdir" - else - echo ERROR: "Overlay is still mounted: $CHROOT_DIR" >&2 - echo ERROR: "Cannot delete: $overlay" >&2 - echo ERROR: "Cannot delete: $upperdir" >&2 - echo ERROR: "Cannot delete: $workdir" >&2 - return 1 - fi -} - check_external_mounts() { - echo "$EXTERNAL_MOUNTS" | sed '/^[[:space:]]*$/d' | while read -r mountexpr ; do + echo "$EXTERNAL_MOUNTS" | while read -r mountexpr ; do + #Skip empty lines, done with if for readability. + if [ -z $mountexpr ]; then + continue + fi mount_type=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $1;}') external=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $2;}') - internal=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $3;}' | sed -e 's#^/*##' -e 's#//*#/#g') + internal=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $3;}') case "$mount_type" in ro|rw) @@ -203,16 +165,13 @@ check_external_mounts() do_external_mounts() { echo INFO: "Bind mounting of external mounts..." >&2 - echo "$EXTERNAL_MOUNTS" | sed '/^[[:space:]]*$/d' | while read -r mountexpr ; do + echo "$EXTERNAL_MOUNTS" | while read -r mountexpr ; do + if [ -z $mountexpr ]; then + continue + fi mount_type=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $1;}') external=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $2;}') - internal=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $3;}' | sed -e 's#^/*##' -e 's#//*#/#g') - - if is_mounted "${CHROOT_DIR}/${internal}" ; then - echo ERROR: "Mountpoint is already mounted: ${CHROOT_DIR}/${internal}" >&2 - echo ERROR: "Fix the issue - cannot proceed" >&2 - exit 1 - fi + internal=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $3;}') # trying to follow the behaviour of docker if ! [ -e "$external" ] || [ -d "$external" ] ; then @@ -242,7 +201,9 @@ do_external_mounts() exit 1 fi - if ! mount --make-rprivate -o bind,${mount_type} "$external" "${CHROOT_DIR}/${internal}" ; then +#Note, this double mounting is needed to support older util-linux. + if ! mount -o bind "${external}" "${CHROOT_DIR}/${internal}" || + ! mount -o remount,bind,${mount_type} "${CHROOT_DIR}/${internal}" ; then echo ERROR: "Failed to mount: ${external} -> ${internal}" >&2 exit 1 else @@ -251,231 +212,166 @@ do_external_mounts() done } -# arg: <mountpoint> -umount_retry() -{ - mountpoint=$(echo "$1" | sed 's#//*#/#g') - timeout=${UMOUNT_TIMEOUT} - - umount "$mountpoint" 2>/dev/null - while is_mounted "$mountpoint" && [ $timeout -gt 0 ] ; do - umount "$mountpoint" 2>/dev/null - sleep 1 - timeout=$(( timeout - 1 )) - done - - if ! is_mounted "$mountpoint" ; then - return 0 - fi - - return 1 -} - -undo_external_mounts() -{ - echo INFO: "Umount external mount points..." >&2 - echo "$EXTERNAL_MOUNTS" | tac | sed '/^[[:space:]]*$/d' | while read -r mountexpr ; do - mount_type=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $1;}') - external=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $2;}') - internal=$(echo "$mountexpr" | awk 'BEGIN{FS=":"}{print $3;}' | sed -e 's#^/*##' -e 's#//*#/#g') - if umount_retry "${CHROOT_DIR}/${internal}" ; then - echo INFO: "Unmounted: ${CHROOT_DIR}/${internal}" >&2 - else - echo ERROR: "Failed to umount: ${CHROOT_DIR}/${internal}" >&2 - fi - done -} - -install_wrapper() -{ - cat > "$CHROOT_DIR"/usr/local/bin/fakeshell.sh <<EOF -#!/bin/sh - -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -export PATH - -gid_tty=\$(getent group | sed -n '/^tty:/p' | cut -d: -f 3) - -mount -t proc proc /proc -mount -t sysfs none /sys -mount -t tmpfs none /dev - -mkdir -p /dev/shm -mkdir -p /dev/pts -mount -t devpts -o gid=\${gid_tty},mode=620 none /dev/pts - -[ -e /dev/full ] || mknod -m 666 /dev/full c 1 7 -[ -e /dev/ptmx ] || mknod -m 666 /dev/ptmx c 5 2 -[ -e /dev/random ] || mknod -m 644 /dev/random c 1 8 -[ -e /dev/urandom ] || mknod -m 644 /dev/urandom c 1 9 -[ -e /dev/zero ] || mknod -m 666 /dev/zero c 1 5 -[ -e /dev/tty ] || mknod -m 666 /dev/tty c 5 0 -[ -e /dev/console ] || mknod -m 622 /dev/console c 5 1 -[ -e /dev/null ] || mknod -m 666 /dev/null c 1 3 - -chown root:tty /dev/console -chown root:tty /dev/ptmx -chown root:tty /dev/tty - -mkdir -p "\$1" || exit 1 -cd "\$1" || exit 1 -shift - -exec "\$@" - -EOF - chmod +x "$CHROOT_DIR"/usr/local/bin/fakeshell.sh -} - -on_exit() -{ - set +e - echo - - if [ -n "$OVERLAY_MOUNT" ] ; then - undo_external_mounts - fi - cleanup -} # -# parse arguments +# parse arguments out of namespace. # -state=nil -action=nil -EXTERNAL_MOUNTS='' -CHROOT_WORKDIR='' -CHROOT_METADIR='' -CHROOT_DIR='' -COMMAND='' -while [ -n "$1" ] ; do - case "$state" in - nil) - case "$1" in - ''|-h|--help|help) - help - exit 0 - ;; - --mount) - EXTERNAL_MOUNTS=$(printf "%s\n%s\n" "$EXTERNAL_MOUNTS" "${2}") - state=next - ;; - --workdir) - if [ -z "$CHROOT_WORKDIR" ] ; then - CHROOT_WORKDIR="$2" +if [ -z $IN_NAMESPACE ]; then + export state=nil + export action=nil + export EXTERNAL_MOUNTS='' + export CHROOT_WORKDIR='' + export CHROOT_METADIR='' + export CHROOT_DIR='' + export COMMAND='' + while [ -n "$1" ] ; do + case "$state" in + nil) + case "$1" in + ''|-h|--help|help) + help + exit 0 + ;; + --mount) + EXTERNAL_MOUNTS=$(printf "%s\n%s" "$EXTERNAL_MOUNTS" "${2}") state=next - else - echo ERROR: "Multiple working directory argument" >&2 + ;; + --workdir) + if [ -z "$CHROOT_WORKDIR" ] ; then + CHROOT_WORKDIR="$2" + state=next + else + echo ERROR: "Multiple working directory argument" >&2 + help >&2 + exit 1 + fi + ;; + execute) + action=execute + state=execute + ;; + *) + echo ERROR: "Bad usage" >&2 help >&2 exit 1 - fi - ;; - execute) - action=execute - state=execute - ;; - *) - echo ERROR: "Bad usage" >&2 - help >&2 - exit 1 - ;; - esac - ;; - next) - state=nil - ;; - execute) - CHROOT_METADIR="$1" - shift - break - ;; - esac - shift -done - - -case "$action" in - ''|nil) + ;; + esac + ;; + next) + state=nil + ;; + execute) + CHROOT_METADIR="$1" + shift + break + ;; + esac + shift + done + + + if [ $action = "nil" ]; then echo ERROR: "Nothing to do - missing command" >&2 help >&2 exit 1 - ;; - execute) - # firstly do sanity checking ... - - if [ -z "$CHROOT_METADIR" ] ; then - echo ERROR: "Missing argument" >&2 - help >&2 - exit 1 - fi - - # making sure that CHROOT_METADIR is absolute path - CHROOT_METADIR=$(readlink -f "$CHROOT_METADIR") - - if ! [ -d "$CHROOT_METADIR"/chroot ] ; then - echo ERROR: "Filepath does not exist: ${CHROOT_METADIR}/chroot" >&2 - exit 1 - fi - - # check external mounts if there are any - check_external_mounts - - # check workdir - if [ -n "$CHROOT_WORKDIR" ] ; then - CHROOT_WORKDIR=$(echo "$CHROOT_WORKDIR" | sed -e 's#^/*##' -e 's#//*#/#g') - fi - - # we must be root - if [ "$(id -u)" -ne 0 ] ; then - echo ERROR: "Need to be root and you are not: $(id -nu)" >&2 - exit 1 - fi + fi - if ! which unshare >/dev/null 2>/dev/null ; then - echo ERROR: "'unshare' system command is missing - ABORT" >&2 - echo INFO: "Try to install 'util-linux' package" >&2 - exit 1 - fi + # do sanity checking ... - # ... sanity checking done + if [ -z "$CHROOT_METADIR" ] ; then + echo ERROR: "Missing argument" >&2 + help >&2 + exit 1 + fi - # setup paths - lowerdir="$CHROOT_METADIR"/chroot - upperdir="$CHROOT_METADIR"/.overlay - workdir="$CHROOT_METADIR"/.workdir - overlay="$CHROOT_METADIR"/.merged + # making sure that CHROOT_METADIR is absolute path + CHROOT_METADIR=$(readlink -f "$CHROOT_METADIR") - # set trap - trap on_exit QUIT TERM EXIT + if ! [ -d "$CHROOT_METADIR"/chroot ] ; then + echo ERROR: "Filepath does not exist: ${CHROOT_METADIR}/chroot" >&2 + exit 1 + fi - # mount overlay - OVERLAY_MOUNT='' - if do_overlay_mount ; then - # overlay chroot - OVERLAY_MOUNT=yes - else - # non overlay mount - OVERLAY_MOUNT=no - fi + # check external mounts if there are any + check_external_mounts - # do the user-specific mounts - do_external_mounts + # we must be root + if [ "$(id -u)" -ne 0 ] ; then + echo ERROR: "Need to be root and you are not: $(id -nu)" >&2 + exit 1 + fi - # I need this wrapper to do some setup inside the chroot... - install_wrapper + if ! which unshare >/dev/null 2>/dev/null ; then + echo ERROR: "'unshare' system command is missing - ABORT" >&2 + echo INFO: "Try to install 'util-linux' package" >&2 + exit 1 + fi - # execute chroot - if [ -n "$1" ] ; then - : - else - set -- /bin/sh -l - fi - unshare -mfpi --propagation private \ - chroot "$CHROOT_DIR" /usr/local/bin/fakeshell.sh "${CHROOT_WORKDIR:-/}" "$@" - ;; -esac + # ... sanity checking done + + #Reexec ourselves in new pid and mount namespace (isolate!). + #Note: newly executed shell will be pid1 in a new namespace. Killing it will kill + #every other process in the whole process tree with sigkill. That will in turn + #destroy namespaces and undo all mounts done previously. + IN_NAMESPACE=1 exec unshare -mpf "$0" "$@" +fi + +#We are namespaced. +# setup paths +lowerdir="$CHROOT_METADIR"/chroot +ovtempdir="$CHROOT_METADIR"/tmp +upperdir="$ovtempdir"/.overlay +workdir="$ovtempdir"/.workdir +overlay="$CHROOT_METADIR"/.merged + +#In case we are using a realy old unshare, make the whole tree into private mounts manually. +mount --make-rprivate / +#New mounts are private always from now on. + +do_overlay_mount + +# do the user-specific mounts +do_external_mounts + +#And setup api filesystems. +mount -t proc proc "${CHROOT_DIR}/proc" +mount -t sysfs none "${CHROOT_DIR}/sys" +mount -t tmpfs none "${CHROOT_DIR}/dev" + +mkdir -p "${CHROOT_DIR}/dev/shm" +mkdir -p "${CHROOT_DIR}/dev/pts" +mount -t devpts none "${CHROOT_DIR}/dev/pts" + +mknod -m 666 "${CHROOT_DIR}/dev/full" c 1 7 +mknod -m 666 "${CHROOT_DIR}/dev/ptmx" c 5 2 +mknod -m 644 "${CHROOT_DIR}/dev/random" c 1 8 +mknod -m 644 "${CHROOT_DIR}/dev/urandom" c 1 9 +mknod -m 666 "${CHROOT_DIR}/dev/zero" c 1 5 +mknod -m 666 "${CHROOT_DIR}/dev/tty" c 5 0 +mknod -m 622 "${CHROOT_DIR}/dev/console" c 5 1 +mknod -m 666 "${CHROOT_DIR}/dev/null" c 1 3 +ln -s /proc/self/fd/0 "$CHROOT_DIR/dev/stdin" +ln -s /proc/self/fd/1 "$CHROOT_DIR/dev/stdout" +ln -s /proc/self/fd/2 "$CHROOT_DIR/dev/stderr" + +# execute chroot +if [ -z "$1" ] ; then + set -- /bin/sh -l +fi + +#The redirection is to save our stdin, because we use it to pipe commands and we +#may want interactivity. +exec chroot "${CHROOT_DIR}" /bin/sh /dev/stdin "${CHROOT_WORKDIR:-/}" "$@" 3<&0 << "EOF" +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +export PATH +mkdir -p $1 +cd $1 +shift +#I intend to reset stdin back *and* close the copy. +exec "$@" <&3 3<&- +EOF exit 0 diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 0d22ac5d..f9d6726f 100755 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -90,9 +90,10 @@ runtime_images: {} # Application specific params # ############################### -# App Helm charts directory location in installation package. -# The path is absolute path (even locates relative inside of this sw package -# installation folder) because it must be visible for ansible docker/chroot +# App Helm charts directory location in installation package +# (local path for the ansible process). +# The path locates relative inside of this sw package +# installation folder and must be visible for ansible docker/chroot # process to find directory and to transfer it into machine (infra node) running # Helm repository. # Content of the folder must be Helm chart directories of the app with Makefile. @@ -100,7 +101,7 @@ runtime_images: {} # NOTE: This default value should not be changed if not really needed and it # must match with the variable "HELM_CHARTS_DIR_IN_PACKAGE" value in package.sh # script! -app_helm_charts_install_directory: "/ansible/application/helm_charts" +app_helm_charts_install_directory: application/helm_charts # Specify target dir where helm charts are copied into on infra node. # (same as content of "app_helm_charts_install_directory" copied by installer to this dir.) @@ -146,3 +147,12 @@ application_post_install_role: # openStackDomain: "Default" # openStackUserName: "admin" # openStackEncryptedPassword: "f7920677e15e2678b0f33736189e8965" + +# Optional time synchronisation settings +# timesync: +# servers: +# - <ip address of NTP_1> +# - <...> +# - <ip address of NTP_N> +# slewclock: false +# timezone: <timezone name from tz database> diff --git a/ansible/group_vars/infrastructure.yml b/ansible/group_vars/infrastructure.yml index 9fd88f25..a248a514 100755 --- a/ansible/group_vars/infrastructure.yml +++ b/ansible/group_vars/infrastructure.yml @@ -24,13 +24,6 @@ simulated_hosts: - registry.npmjs.org all_simulated_hosts: "{{ simulated_hosts.git + simulated_hosts.http + simulated_hosts.nexus }}" -rancher_server_url: "http://{{ hostvars[groups.infrastructure.0].ansible_host }}:8080" rancher_server_version: v1.6.22 -rancher_remove_other_env: yes -rancher_redeploy_k8s_env: yes -populate_nexus: no -kube_directory: /root/.kube -kubectl_bin_dir: /usr/local/bin +populate_nexus: false helm_bin_dir: /usr/local/bin -helm_repository_name: local -helm_repository_url: http://127.0.0.1:8879 diff --git a/ansible/infrastructure.yml b/ansible/infrastructure.yml index 18290ae4..e49ee919 100644 --- a/ansible/infrastructure.yml +++ b/ansible/infrastructure.yml @@ -3,11 +3,13 @@ hosts: infrastructure, kubernetes roles: - package-repository - - role: firewall + - firewall - name: Setup infrastructure servers hosts: infrastructure roles: + - chrony + - package-repository-check - certificates - docker - dns @@ -18,6 +20,8 @@ - name: Setup base for Kubernetes nodes hosts: kubernetes roles: + - chrony + - package-repository-check - docker tasks: - include_role: diff --git a/ansible/rancher_kubernetes.yml b/ansible/rancher_kubernetes.yml index 196f1fc2..e44f18d3 100644 --- a/ansible/rancher_kubernetes.yml +++ b/ansible/rancher_kubernetes.yml @@ -1,31 +1,23 @@ --- -- name: Install binaries for controlling deployment +- name: Deploy rancher server and create k8s env hosts: infrastructure roles: + - role: rancher + vars: + mode: server - kubectl - helm -- name: Deploy rancher server and create k8s env - hosts: infrastructure - roles: - - rancher - vars: - rancher_role: server - -- name: Deploy rancher agents +- name: Deploy rancher agents for kube nodes hosts: kubernetes roles: - - rancher - vars: - rancher_role: agent + - role: rancher + vars: + mode: agent - name: Wait for Kubernetes environment to be healthy hosts: infrastructure - tasks: - - name: Check cluster health - uri: - url: "{{ rancher_server_url }}/v2-beta/projects/{{ k8s_env_id }}" - register: env_info - retries: 30 - delay: 15 - until: "env_info.json.healthState == 'healthy'" + roles: + - role: rancher + vars: + mode: health
\ No newline at end of file diff --git a/ansible/roles/application-install/defaults/main.yml b/ansible/roles/application-install/defaults/main.yml deleted file mode 100644 index 473fbb80..00000000 --- a/ansible/roles/application-install/defaults/main.yml +++ /dev/null @@ -1 +0,0 @@ -phase: install diff --git a/ansible/roles/application-install/tasks/main.yml b/ansible/roles/application-install/tasks/main.yml deleted file mode 100644 index ba522792..00000000 --- a/ansible/roles/application-install/tasks/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- debug: - msg: "phase is {{ phase }}" - -- name: Check if install needed - block: - - name: "Does {{ app_helm_charts_install_directory }} exist and contain Helm Charts" - find: - paths: "{{ app_helm_charts_install_directory }}" - recurse: yes - delegate_to: localhost - register: charts_files - - name: Set install active fact - set_fact: - install_needed: "{{ true if charts_files.matched | int > 0 else false }}" - when: phase == "pre-install" - -- include_tasks: "{{ phase }}.yml" - when: install_needed - -- debug: - msg: "Install needed {{ install_needed }}" diff --git a/ansible/roles/application-override/tasks/main.yml b/ansible/roles/application-override/tasks/main.yml new file mode 100644 index 00000000..1ecf7c79 --- /dev/null +++ b/ansible/roles/application-override/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Role for generating Helm override.yml file +- name: Register root certificate + slurp: + src: "{{ playbook_dir }}/certs/rootCA.crt" + register: root_cert + delegate_to: localhost + +- name: "Set root ca certificate" + set_fact: + merged_overrides: "{{ overrides | default({}) | combine({'global': {'cacert': root_cert.content | b64decode}}, recursive=True) }}" + +- name: "Create {{ app_helm_override_file }}" + copy: + dest: "{{ app_helm_override_file }}" + content: "{{ merged_overrides | to_nice_yaml }}" diff --git a/ansible/roles/application/defaults/main.yml b/ansible/roles/application/defaults/main.yml new file mode 100644 index 00000000..dec17601 --- /dev/null +++ b/ansible/roles/application/defaults/main.yml @@ -0,0 +1,11 @@ +--- +helm_repository_name: local +helm_repository_url: http://127.0.0.1:8879 +# Override file generation for Helm application can be customized by any role +# given by user and found by ansible from roles_path. +# By default override file is generated by 'application-override' role that is +# specific for offline installer (for onap) as it's generating server +# certificate needed to simulate internet by offline installer. +app_skip_helm_override: false +app_helm_override_role: application-override +app_helm_override_file: "{{ app_data_path }}/override.yaml" diff --git a/ansible/roles/application-install/tasks/custom_role.yml b/ansible/roles/application/tasks/custom_role.yml index b6f6f351..f0b9a84f 100644 --- a/ansible/roles/application-install/tasks/custom_role.yml +++ b/ansible/roles/application/tasks/custom_role.yml @@ -1,6 +1,6 @@ --- # Caller fills application_custom_role variable with actual role name. -- name: "Execute custom role {{ application_custom_role }} {{ phase }} Helm install." +- name: "Execute custom role {{ application_custom_role }} for Helm install." include_role: name: "{{ application_custom_role }}" when: diff --git a/ansible/roles/application-install/tasks/install.yml b/ansible/roles/application/tasks/install.yml index 1cccf9ad..103ecc8b 100644 --- a/ansible/roles/application-install/tasks/install.yml +++ b/ansible/roles/application/tasks/install.yml @@ -6,8 +6,8 @@ --skip-refresh changed_when: true # init is always changed type of action -#A correct way to implement this would be using --wait option in helm init invocation. -#However, it does not work due to https://github.com/helm/helm/issues/4031 (fixed in newer helm release) +# A correct way to implement this would be using --wait option in helm init invocation. +# However, it does not work due to https://github.com/helm/helm/issues/4031 (fixed in newer helm release) - name: "Wait for helm upgrade to finish" command: "{{ helm_bin_dir }}/helm version --tiller-connection-timeout 10" register: result @@ -42,7 +42,7 @@ - "'Error: no repositories to show' not in helm_repo_list.stderr" - name: Helm Add Repo - command: "{{ helm_bin_dir }}/helm repo add {{ helm_repository_name }} {{ helm_repository_url }}" + command: "{{ helm_bin_dir }}/helm repo add {{ helm_repository_name | mandatory }} {{ helm_repository_url | mandatory }}" when: "'local' not in helm_repo_list.stdout" changed_when: true # when executed its a changed type of action @@ -50,24 +50,14 @@ make: chdir: "{{ app_helm_charts_infra_directory }}" target: "{{ item }}" - with_items: "{{ app_helm_build_targets }}" + loop: "{{ app_helm_build_targets }}" environment: PATH: "{{ helm_bin_dir }}:{{ ansible_env.PATH }}" -- name: Register root certificate - slurp: - src: "{{ playbook_dir }}/certs/rootCA.crt" - register: root_cert - delegate_to: localhost - -# WA: this is required because deploy plugin dont process params properly -- name: Create override file with global.cacert - copy: - dest: "{{ app_data_path }}/override.yaml" - content: | - global: - cacert: | - {{ root_cert['content'] | b64decode | indent( width=4, indentfirst=False) }} +- name: Generate Helm application override file with custom role + include_role: + name: "{{ app_helm_override_role }}" + when: not app_skip_helm_override - name: Check for deploy plugin presence stat: @@ -81,7 +71,7 @@ {{ app_helm_release_name }} {{ helm_repository_name }}/{{ app_helm_chart_name }} --namespace {{ app_kubernetes_namespace }} - -f {{ app_data_path }}/override.yaml + {{ '' if app_skip_helm_override else '-f ' + app_helm_override_file }} changed_when: true # when executed its a changed type of action register: helm_install failed_when: helm_install.stderr diff --git a/ansible/roles/application/tasks/main.yml b/ansible/roles/application/tasks/main.yml new file mode 100644 index 00000000..3018e95f --- /dev/null +++ b/ansible/roles/application/tasks/main.yml @@ -0,0 +1,24 @@ +--- +- name: Check if application Helm charts exist and install is even needed + block: + - name: "Does {{ app_helm_charts_install_directory }} directory exist and contain Helm Charts" + find: + paths: "{{ app_helm_charts_install_directory }}" + recurse: true + delegate_to: localhost + register: charts_files + - name: Set install active fact + set_fact: + install_needed: "{{ true if charts_files.matched | int > 0 else false }}" + +- name: Install app with Helm charts + block: + - include_tasks: transfer-helm-charts.yml + - include_tasks: pre-install.yml + - include_tasks: install.yml + - include_tasks: post-install.yml + when: install_needed + +- debug: + msg: "NOTE, nothing done as application Helm charts does not exist!" + when: not install_needed diff --git a/ansible/roles/application-install/tasks/post-install.yml b/ansible/roles/application/tasks/post-install.yml index 10594233..5464cb46 100644 --- a/ansible/roles/application-install/tasks/post-install.yml +++ b/ansible/roles/application/tasks/post-install.yml @@ -2,4 +2,4 @@ - name: "Execute custome role {{ application_post_install_role }} if defined." include_tasks: custom_role.yml vars: - application_custom_role: "{{ application_post_install_role }}" + application_custom_role: "{{ application_post_install_role | default('') }}" diff --git a/ansible/roles/application/tasks/pre-install.yml b/ansible/roles/application/tasks/pre-install.yml new file mode 100644 index 00000000..74f1548f --- /dev/null +++ b/ansible/roles/application/tasks/pre-install.yml @@ -0,0 +1,5 @@ +--- +- name: "Execute custom role {{ application_pre_install_role }} if defined." + include_tasks: custom_role.yml + vars: + application_custom_role: "{{ application_pre_install_role | default('') }}" diff --git a/ansible/roles/application-install/tasks/pre-install.yml b/ansible/roles/application/tasks/transfer-helm-charts.yml index bf6619b0..0cd7c02f 100644 --- a/ansible/roles/application-install/tasks/pre-install.yml +++ b/ansible/roles/application/tasks/transfer-helm-charts.yml @@ -38,14 +38,7 @@ copy: src: "{{ item.path }}" dest: "{{ helm_home_dir.stdout }}/plugins" - directory_mode: yes + directory_mode: true mode: 0755 with_items: "{{ list_of_plugins.files }}" - become: true when: app_helm_plugins_directory is defined and app_helm_plugins_directory is not none - -- name: "Execute custom role {{ application_pre_install_role }} if defined." - include_tasks: custom_role.yml - vars: - application_custom_role: "{{ application_pre_install_role }}" - diff --git a/ansible/roles/chrony/defaults/main.yml b/ansible/roles/chrony/defaults/main.yml new file mode 100644 index 00000000..af433dac --- /dev/null +++ b/ansible/roles/chrony/defaults/main.yml @@ -0,0 +1,16 @@ +--- +timesync: {} +chrony: + servers: "{{ timesync.servers | default([hostvars[groups.infrastructure[0]].cluster_ip]) }}" # chronyd's NTP servers + slewclock: "{{ timesync.slewclock | default(false) }}" # chronyd's makestep property + timezone: "{{ timesync.timezone | default('Universal') }}" # Timezone name according to tz database + makestep: '1 -1' + maxjitter: 10 # Max allowed jitter if using infra as time source as it may by unstable due to pretending stratum 1 time source + initstepslew: 30 + conf: + RedHat: + config_file: /etc/chrony.conf + driftfile: /var/lib/chrony/drift + Debian: + config_file: /etc/chrony/chrony.conf + driftfile: /var/lib/chrony/chrony.drift diff --git a/ansible/roles/chrony/handlers/main.yml b/ansible/roles/chrony/handlers/main.yml new file mode 100644 index 00000000..80ab9fa9 --- /dev/null +++ b/ansible/roles/chrony/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart chronyd + systemd: + name: chronyd + state: restarted diff --git a/ansible/roles/chrony/tasks/main.yml b/ansible/roles/chrony/tasks/main.yml new file mode 100644 index 00000000..69a11587 --- /dev/null +++ b/ansible/roles/chrony/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: Check if server mode + set_fact: + chrony_mode: 'server' + when: "'infrastructure' in group_names and timesync.servers is not defined" + +- name: Check if client mode + set_fact: + chrony_mode: 'client' + when: "timesync.servers is defined or 'infrastructure' not in group_names" + +- name: "Upload chronyd {{ chrony_mode }} configuration" + template: + src: "chrony.conf.j2" + dest: "{{ chrony['conf'][ansible_os_family]['config_file'] }}" + notify: Restart chronyd + +- name: Ensure chronyd is enabled/running + systemd: + name: chronyd + state: started + enabled: true + +- name: Setup timezone + timezone: + name: "{{ chrony.timezone }}" diff --git a/ansible/roles/chrony/templates/chrony.conf.j2 b/ansible/roles/chrony/templates/chrony.conf.j2 new file mode 100644 index 00000000..3bfb4e40 --- /dev/null +++ b/ansible/roles/chrony/templates/chrony.conf.j2 @@ -0,0 +1,22 @@ +{% if chrony_mode == 'server' %} +local stratum 1 +allow +{% elif chrony_mode == 'client' %} +{% for tserver in chrony.servers %} +server {{ tserver }} iburst +{% endfor %} +{% if chrony.slewclock == false %} +{# Step the time by default #} +makestep {{ chrony.makestep }} +{% else %} +{# Slew the clock but step at boot time if time error larger than 30 seconds #} +initstepslew {{ chrony.initstepslew }}{% for tserver in chrony.servers %} {{ tserver }}{% endfor %} + +{% endif %} +{% if timesync.servers is not defined %} +maxjitter {{ chrony.maxjitter }} +{% endif %} +{% endif %} +driftfile {{ chrony['conf'][ansible_os_family]['driftfile'] }} +rtcsync +logdir /var/log/chrony diff --git a/ansible/roles/docker/handlers/docker_restart.yml b/ansible/roles/docker/handlers/docker_restart.yml new file mode 100644 index 00000000..8feb9a95 --- /dev/null +++ b/ansible/roles/docker/handlers/docker_restart.yml @@ -0,0 +1,18 @@ +--- +- name: Stopping docker systemd service + systemd: + name: docker + state: stopped + enabled: true + +# WA for docker bug, see OOM-1735 +- name: Ensure docker.sock cleaned properly + file: + state: absent + path: /var/run/docker.sock + +- name: Starting docker systemd service + systemd: + name: docker + state: started + enabled: true diff --git a/ansible/roles/docker/handlers/main.yml b/ansible/roles/docker/handlers/main.yml index 5df47e8d..de4a52ac 100644 --- a/ansible/roles/docker/handlers/main.yml +++ b/ansible/roles/docker/handlers/main.yml @@ -1,6 +1,3 @@ --- - name: Restart Docker - systemd: - name: docker - state: restarted - enabled: yes + import_tasks: docker_restart.yml diff --git a/ansible/roles/kubectl/defaults/main.yml b/ansible/roles/kubectl/defaults/main.yml new file mode 100644 index 00000000..78c15c75 --- /dev/null +++ b/ansible/roles/kubectl/defaults/main.yml @@ -0,0 +1,5 @@ +--- +kubectl_bin_dir: /usr/local/bin +kube_directory: ~/.kube +# Defaulting to rancher setup +kube_server: "https://{{ ansible_host }}:8080/r/projects/{{ k8s_env_id | mandatory }}/kubernetes:6443"
\ No newline at end of file diff --git a/ansible/roles/kubectl/tasks/main.yml b/ansible/roles/kubectl/tasks/main.yml index 7c77c3c5..9ecb5c44 100644 --- a/ansible/roles/kubectl/tasks/main.yml +++ b/ansible/roles/kubectl/tasks/main.yml @@ -5,3 +5,13 @@ dest: "{{ kubectl_bin_dir }}/kubectl" remote_src: true mode: 0755 + +- name: Ensure .kube directory exists + file: + path: "{{ kube_directory }}" + state: directory + +- name: Create kube config + template: + src: kube_config.j2 + dest: "{{ kube_directory }}/config" diff --git a/ansible/roles/rancher/templates/kube_config.j2 b/ansible/roles/kubectl/templates/kube_config.j2 index 87f332e6..586c59d4 100644 --- a/ansible/roles/rancher/templates/kube_config.j2 +++ b/ansible/roles/kubectl/templates/kube_config.j2 @@ -4,7 +4,7 @@ clusters: - cluster: api-version: v1 insecure-skip-tls-verify: true - server: "https://{{ ansible_host }}:8080/r/projects/{{ k8s_env_id }}/kubernetes:6443" + server: "{{ kube_server }}" name: "{{ app_name }}" contexts: - context: diff --git a/ansible/roles/nginx/templates/nginx.conf.j2 b/ansible/roles/nginx/templates/nginx.conf.j2 index ff9d2a9c..9860a168 100644 --- a/ansible/roles/nginx/templates/nginx.conf.j2 +++ b/ansible/roles/nginx/templates/nginx.conf.j2 @@ -5,6 +5,7 @@ events { } http { + server_names_hash_bucket_size 64; error_log /var/log/nginx/error.log debug; access_log /var/log/nginx/access.log; diff --git a/ansible/roles/package-repository-check/tasks/RedHat.yml b/ansible/roles/package-repository-check/tasks/RedHat.yml new file mode 100644 index 00000000..ed496f99 --- /dev/null +++ b/ansible/roles/package-repository-check/tasks/RedHat.yml @@ -0,0 +1,20 @@ +--- +- name: verify + block: + # Clean cache prior to refreshing + - name: Clean yum cache + command: yum clean all + args: + warn: false + # Refresh cache to ensure repo is reachable + - name: Update yum cache + yum: + update_cache: yes + state: latest + tags: + - skip_ansible_lint # Prevent '[403] Package installs should not use latest' ansible lint task rule + rescue: + - name: Fail if yum cache updating failed + fail: + msg: "Couldn't refresh yum cache, repositories not configured properly. Check ansible logs for details." + become: true diff --git a/ansible/roles/package-repository-check/tasks/main.yml b/ansible/roles/package-repository-check/tasks/main.yml new file mode 100644 index 00000000..ac63eceb --- /dev/null +++ b/ansible/roles/package-repository-check/tasks/main.yml @@ -0,0 +1,12 @@ +--- +# Purpose of this role is to check whether configured repositories are working. +# +# Successfull prior invocation of 'package-repository' role on 'infrastructure' hosts +# is prerequisite for playing this one on 'infrastructure' group. +# +# Successfull prior invocation of 'package-repository' and 'nginx' role on infrastructure hosts +# is prerequisite for playing this one on 'kubernetes' group. + +#Set of tasks designated to failing fast if configured repos are not functioning properly +- include_tasks: "{{ ansible_os_family }}.yml" + when: ansible_os_family == 'RedHat' diff --git a/ansible/roles/rancher/defaults/main.yml b/ansible/roles/rancher/defaults/main.yml new file mode 100644 index 00000000..8edc5180 --- /dev/null +++ b/ansible/roles/rancher/defaults/main.yml @@ -0,0 +1,4 @@ +--- +rancher_server_url: "http://{{ hostvars[groups.infrastructure.0].ansible_host }}:8080" +rancher_remove_other_env: true +rancher_redeploy_k8s_env: true diff --git a/ansible/roles/rancher/tasks/main.yml b/ansible/roles/rancher/tasks/main.yml index 1370a39f..045363d0 100644 --- a/ansible/roles/rancher/tasks/main.yml +++ b/ansible/roles/rancher/tasks/main.yml @@ -1,2 +1,2 @@ --- -- include_tasks: "rancher_{{ rancher_role }}.yml" +- include_tasks: "rancher_{{ mode }}.yml" diff --git a/ansible/roles/rancher/tasks/rancher_agent.yml b/ansible/roles/rancher/tasks/rancher_agent.yml index 4c9cb8dd..091503c7 100644 --- a/ansible/roles/rancher/tasks/rancher_agent.yml +++ b/ansible/roles/rancher/tasks/rancher_agent.yml @@ -7,7 +7,7 @@ volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "/var/lib/rancher:/var/lib/rancher" - auto_remove: yes - privileged: yes + auto_remove: true + privileged: true vars: server_hostvars: "{{ hostvars[groups.infrastructure.0] }}" diff --git a/ansible/roles/rancher/tasks/rancher_health.yml b/ansible/roles/rancher/tasks/rancher_health.yml new file mode 100644 index 00000000..b0323739 --- /dev/null +++ b/ansible/roles/rancher/tasks/rancher_health.yml @@ -0,0 +1,8 @@ +--- +- name: Check cluster health + uri: + url: "{{ rancher_server_url }}/v2-beta/projects/{{ k8s_env_id }}" + register: env_info + retries: 30 + delay: 15 + until: "env_info.json.healthState == 'healthy'" diff --git a/ansible/roles/rancher/tasks/rancher_server.yml b/ansible/roles/rancher/tasks/rancher_server.yml index 64b35e4c..ebee6cc7 100644 --- a/ansible/roles/rancher/tasks/rancher_server.yml +++ b/ansible/roles/rancher/tasks/rancher_server.yml @@ -2,7 +2,7 @@ # DO NOT ADD SPACE AROUND ';' - name: Start rancher/server:{{ rancher_server_version }} docker_container: - name: rancher_server + name: rancher-server image: rancher/server:{{ rancher_server_version }} command: ["sh", "-c", "/usr/sbin/update-ca-certificates;/usr/bin/entry /usr/bin/s6-svscan /service"] ports: 8080:8080 @@ -19,7 +19,7 @@ delay: 30 until: not response.failed -- name: Create kubernetes environment +- name: Create rancher kubernetes environment rancher_k8s_environment: name: "{{ app_name }}" descr: "Kubernetes environment for {{ app_name }}" @@ -39,13 +39,3 @@ key_private: "{{ env.data.apikey.private }}" rancher_agent_image: "{{ env.data.registration_tokens.image }}" rancher_agent_reg_url: "{{ env.data.registration_tokens.reg_url }}" - -- name: Ensure .kube directory exists - file: - path: "{{ kube_directory }}" - state: directory - -- name: Create kube config - template: - src: kube_config.j2 - dest: "{{ kube_directory }}/config" diff --git a/ansible/roles/resource-data/tasks/main.yml b/ansible/roles/resource-data/tasks/main.yml index 41046d81..023a160e 100644 --- a/ansible/roles/resource-data/tasks/main.yml +++ b/ansible/roles/resource-data/tasks/main.yml @@ -3,23 +3,34 @@ block: - name: Check if source dir and files are present stat: - path: "{{ item.source }}" + path: "{{ item.source_dir }}/{{ item.source_filename }}" get_checksum: false loop: - - { source: "{{ resources_dir }}/{{ resources_filename | default('thisdoesnotexists', true) }}", - target: "{{ app_data_path }}/{{ resources_filename | default('thisdoesnotexists', true) }}" } - - { source: "{{ resources_dir }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}", - target: "{{ aux_data_path }}/{{ aux_resources_filename | default('thisdoesnotexists', true) }}" } + - { source_dir: "{{ resources_dir | default('', true) }}", + source_filename: "{{ resources_filename | default('', true) }}", + target_dir: "{{ app_data_path | default('', true) }}", + target_filename: "{{ resources_filename | default('', true) }}" } + + - { source_dir: "{{ resources_dir | default('', true) }}", + source_filename: "{{ aux_resources_filename | default('', true) }}", + target_dir: "{{ aux_data_path | default('', true) }}", + target_filename: "{{ aux_resources_filename | default('', true) }}" } register: source_path + when: + - item.source_dir | length > 0 + - item.source_filename | length > 0 + - item.target_dir | length > 0 + - item.target_filename | length > 0 - name: Create initial resources list of dicts set_fact: to_be_uploaded_resources_list: "{{ to_be_uploaded_resources_list | default([]) + [ - {'file': item.item.target | basename, - 'destination_dir': item.item.target | dirname } ] }}" + {'file': item.item.target_filename, + 'destination_dir': item.item.target_dir } ] }}" loop: "{{ source_path.results }}" - when: item.stat.exists + when: item.stat is defined + failed_when: not item.stat.exists when: inventory_hostname in groups.resources - name: "Upload resource files {{ hostvars[groups.resources.0].to_be_uploaded_resources_list }} to infrastructure" diff --git a/ansible/test/images/docker/build-all.sh b/ansible/test/images/docker/build-all.sh new file mode 100755 index 00000000..dd5db093 --- /dev/null +++ b/ansible/test/images/docker/build-all.sh @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +BUILD_SCRIPT=${1:-build.sh} + +# Run all build scripts in direct subdirectories +for buildfile in $(find -mindepth 2 -maxdepth 2 -name ${BUILD_SCRIPT}); +do + pushd $(dirname ${buildfile}) + . ${BUILD_SCRIPT} + popd +done diff --git a/ansible/test/images/docker/centos7/Dockerfile b/ansible/test/images/docker/centos7/Dockerfile new file mode 100644 index 00000000..8c024ab9 --- /dev/null +++ b/ansible/test/images/docker/centos7/Dockerfile @@ -0,0 +1,29 @@ +# https://github.com/chrismeyersfsu/provision_docker/tree/master/files +FROM centos:centos7 +ENV container docker + +RUN yum -y update; yum clean all + +RUN systemctl mask dev-mqueue.mount dev-hugepages.mount \ + systemd-remount-fs.service sys-kernel-config.mount \ + sys-kernel-debug.mount sys-fs-fuse-connections.mount \ + display-manager.service graphical.target systemd-logind.service + +RUN yum -y install openssh-server sudo openssh-clients \ + epel-release python-docker-py iproute +RUN sed -i 's/#PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN ssh-keygen -q -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa && \ + ssh-keygen -q -f /etc/ssh/ssh_host_ecdsa_key -N '' -t ecdsa && \ + ssh-keygen -q -f /etc/ssh/ssh_host_ed25519_key -N '' -t ed25519 +RUN echo 'root:docker.io' | chpasswd +RUN systemctl enable sshd.service + +# firewalld needs this .. and I needs my firewalld +ADD dbus.service /etc/systemd/system/dbus.service +RUN systemctl enable dbus.service + +VOLUME ["/run"] + +EXPOSE 22 + +CMD ["/usr/sbin/init"] diff --git a/ansible/test/images/docker/centos7/build.sh b/ansible/test/images/docker/centos7/build.sh new file mode 100755 index 00000000..fe0aea66 --- /dev/null +++ b/ansible/test/images/docker/centos7/build.sh @@ -0,0 +1,22 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +set -e +docker build . -t molecule-centos7:latest diff --git a/ansible/test/images/docker/centos7/dbus.service b/ansible/test/images/docker/centos7/dbus.service new file mode 100644 index 00000000..69dbb153 --- /dev/null +++ b/ansible/test/images/docker/centos7/dbus.service @@ -0,0 +1,16 @@ +[Unit] +Description=D-Bus System Message Bus +Requires=dbus.socket +After=syslog.target + +[Service] +PIDFile=/var/run/messagebus.pid +ExecStartPre=/bin/mkdir -p /var/run/dbus +ExecStartPre=/bin/chmod g+w /var/run/ /var/run/dbus/ +ExecStart=/bin/dbus-daemon --system --fork +ExecReload=/bin/dbus-send --print-reply --system --type=method_call --dest=org.freedesktop.DBus / org.freedesktop.DBus.ReloadConfig +ExecStopPost=/bin/rm -f /var/run/messagebus.pid +#OOMScoreAdjust=-900 +User=dbus +Group=root +PermissionsStartOnly=true |