summaryrefslogtreecommitdiffstats
path: root/docs/sections/healthcheck.rst
blob: fa017f006af269058c3d23be4430b38842760964 (plain) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

.. This work is licensed under a Creative Commons Attribution 4.0 International License.
.. http://creativecommons.org/licenses/by/4.0

DCAE Health Check
=================

OOM Deployment
--------------
In OOM deployments, DCAE healthchecks are performed by a separate service--dcae-healthcheck.
This service is packaged into a Docker image (``onap/org.onap.dcaegen2.deployments.healthcheck-container``),
which is built in the ``healthcheck-container`` module in the ``dcaegen2/deployments`` repository.

The service is deployed with a Helm chart (``oom/kubernetes/dcaegen2/charts/dcae-healthcheck``)
when DCAE is deployed using OOM.

The dcae-healthcheck container runs a service that exposes a simple Web API.  In response to
request, the service checks Kubernetes to verify that all of the expected
DCAE platform and service components are in a ready state.  The service
has a fixed list of platform and service components that are normally deployed when DCAE is
first installed, including components deployed with Helm charts and
components deployed using Cloudify blueprints.   In addition, the healthcheck
service tracks and checks components that are deployed dynamically using Cloudify
blueprints after the initial DCAE installation.

The healthcheck service is exposed as a Kubernetes ClusterIP Service named
`dcae-healthcheck`.   The service can be queried for status as shown below.

.. code-block:: json

   $ curl dcae-healthcheck
   {
     "type": "summary",
     "count": 14,
     "ready": 14,
     "items": [
        {
          "name": "dev-dcaegen2-dcae-cloudify-manager",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dev-dcaegen2-dcae-config-binding-service",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dev-dcaegen2-dcae-inventory-api",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dev-dcaegen2-dcae-servicechange-handler",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dev-dcaegen2-dcae-deployment-handler",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dev-dcaegen2-dcae-policy-handler",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-ves-collector",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-tca-analytics",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-prh",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-hv-ves-collector",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-dashboard",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-dcae-snmptrap-collector",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-holmes-engine-mgmt",
          "ready": 1,
          "unavailable": 0
        },
        {
          "name": "dep-holmes-rule-mgmt",
          "ready": 1,
          "unavailable": 0
        }
      ]
    }
: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */ .highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */ .highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */ .highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */ .highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */ .highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */ .highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */ .highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */ .highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */ .highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */ .highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */ .highlight .vc { color: #336699 } /* Name.Variable.Class */ .highlight .vg { color: #dd7700 } /* Name.Variable.Global */ .highlight .vi { color: #3333bb } /* Name.Variable.Instance */ .highlight .vm { color: #336699 } /* Name.Variable.Magic */ .highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */ } 
#!/bin/bash

source /var/onap/commons
source /var/onap/config/env-vars
source /var/onap/_composed_functions
source /var/onap/_onap_functions

export MTU=$(/sbin/ifconfig | grep MTU | sed 's/.*MTU://' | sed 's/ .*//' |sort -n | head -1)
export NIC=$(ip route get 8.8.8.8 | awk '{ print $5; exit }')
export IP_ADDRESS=$(ifconfig $NIC | grep "inet addr" | tr -s ' ' | cut -d' ' -f3 | cut -d':' -f2)

mvn_conf_file=/root/.m2/settings.xml

# configure_dns() - DNS/GW IP address configuration
function configure_dns {
    echo "nameserver 10.0.0.1" >> /etc/resolvconf/resolv.conf.d/head
    resolvconf -u
}

# get_next_ip() - Function that provides the next ip
function get_next_ip {
    local ip=${1:-$IP_ADDRESS}
    ip_hex=$(printf '%.2X%.2X%.2X%.2X\n' `echo $ip | sed -e 's/\./ /g'`)
    next_ip_hex=$(printf %.8X `echo $(( 0x$ip_hex + 1 ))`)
    echo $(printf '%d.%d.%d.%d\n' `echo $next_ip_hex | sed -r 's/(..)/0x\1 /g'`)
}

# _git_timed() - git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end.  This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang.
function _git_timed {
    local count=0
    local timeout=0

    install_package git
    until timeout -s SIGINT ${timeout} git "$@"; do
        # 124 is timeout(1)'s special return code when it reached the
        # timeout; otherwise assume fatal failure
        if [[ $? -ne 124 ]]; then
            exit 1
        fi

        count=$(($count + 1))
        if [ $count -eq 3 ]; then
            exit 1
        fi
        sleep 5
    done
}

# clone_repo() - Clone Git repository into specific folder
function clone_repo {
    local repo_url=${3:-"https://git.onap.org/"}
    local repo=$1
    local dest_folder=${2:-$git_src_folder/$repo}
    if [ ! -d $dest_folder ]; then
        if [[ "$debug" == "False" ]]; then
            _git_timed clone --quiet ${repo_url}${repo} $dest_folder
        else
            _git_timed clone ${repo_url}${repo} $dest_folder
        fi
    else
        pushd $dest_folder
        _git_timed pull
        popd
    fi
}

# clone_repos() - Function that clones source repositories for a given project
function clone_repos {
    local project=$1
    local repo_name=${2:-$project}

    for repo in ${repos[$project]}; do
        clone_repo $repo ${src_folders[$project]}${repo#*$repo_name}
    done
}

# _install_bind() - Install bind utils
function _install_bind {
    install_packages bind9 bind9utils
}

# install_java() - Install java binaries
function install_java {
    if is_package_installed openjdk-8-jdk; then
        return
    fi
    source /etc/os-release || source /usr/lib/os-release
    case ${ID,,} in
        *suse)
        ;;
        ubuntu|debian)
            install_package software-properties-common
            add-apt-repository -y ppa:openjdk-r/ppa
        ;;
        rhel|centos|fedora)
        ;;
    esac
    update_repos

    # Remove Java 7
    uninstall_packages default-jre openjdk-7-jdk openjdk-7-jre openjdk-7-jre-headless

    install_package openjdk-8-jdk
    # ca-certificates-java is not a dependency in the Oracle JDK/JRE so this must be explicitly installed.
    /var/lib/dpkg/info/ca-certificates-java.postinst configure
}

# install_maven() - Install maven binaries
function install_maven {
    if is_package_installed maven3; then
        return
    fi
    install_java
    source /etc/os-release || source /usr/lib/os-release
    case ${ID,,} in
        *suse)
        ;;
        ubuntu|debian)
            install_package software-properties-common
            add-apt-repository -y ppa:andrei-pozolotin/maven3
        ;;
        rhel|centos|fedora)
        ;;
    esac
    update_repos
    install_package maven3

    # Remove Java 7
    uninstall_package openjdk-7-jdk

    _configure_maven
}

# _configure_docker_settings() - Configures Docker settings
function _configure_docker_settings {
    local docker_conf_backup=/tmp/docker.backup
    local docker_conf=/etc/default/docker
    local chameleonsocks_filename=chameleonsocks.sh
    local max_concurrent_downloads=${1:-3}

    cp ${docker_conf} ${docker_conf_backup}
    if [ $http_proxy ]; then
        echo "export http_proxy=$http_proxy" >> $docker_conf
    fi
    if [ $https_proxy ]; then
        echo "export https_proxy=$https_proxy" >> $docker_conf
        #If you have a socks proxy, then use that to connect to the nexus repo
        #via a redsocks container
        if [ $socks_proxy ]; then
            wget https://raw.githubusercontent.com/crops/chameleonsocks/master/$chameleonsocks_filename
            chmod 755 $chameleonsocks_filename
            socks=$(echo $socks_proxy | sed -e "s/^.*\///" | sed -e "s/:.*$//")
            port=$(echo $socks_proxy | sed -e "s/^.*://")
            PROXY=$socks PORT=$port ./$chameleonsocks_filename --install
            rm $chameleonsocks_filename
            cp ${docker_conf_backup} ${docker_conf}
        fi
    fi
    rm ${docker_conf_backup}

    echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" >> $docker_conf
    usermod -aG docker $USER

    source /etc/os-release || source /usr/lib/os-release
    case ${ID,,} in
        *suse)
        ;;
        ubuntu|debian)
            service docker restart
            sleep 10
        ;;
        rhel|centos|fedora)
        ;;
    esac
}

# install_nodejs() - Download and install NodeJS
function install_nodejs {
    if is_package_installed nodejs; then
        return
    fi
    curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
    install_package nodejs

    # Update NPM to latest version
    npm install npm -g
}

# install_python() - Install Python 2.7 and other tools necessary for development.
function install_python {
    install_packages python2.7 python-dev
}

# _install_pip() - Install Python Package Manager
function _install_pip {
    install_python
    if ! which pip; then
        curl -sL https://bootstrap.pypa.io/get-pip.py | python
    fi
}

# install_python_package() - Install python modules
function install_python_package {
    local python_packages=$@

    _install_pip
    pip install $python_packages
}

# install_python_requirements() - Install a list of python modules defined in requirement.txt file
function install_python_requirements {
    local python_project_path=$1

    _install_pip
    pushd $python_project_path
    pip install -r requirements.txt
    popd
}

# install_docker() - Download and install docker-engine
function install_docker {
    if $(docker version &>/dev/null); then
        return
    fi
    source /etc/os-release || source /usr/lib/os-release
    case ${ID,,} in
        *suse)
        ;;
        ubuntu|debian)
            install_packages software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl
            curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
            add-apt-repository \
            "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
            $(lsb_release -cs) stable"
        ;;
        rhel|centos|fedora)
        ;;
    esac
    update_repos

    install_package docker-ce
    _configure_docker_settings
}

# pull_docker_image() - Pull Docker container image from the Public Docker Registry Hub
function pull_docker_image {
    install_docker
    local image=$1
    local tag=$2
    docker pull ${image}
    if [ ${tag} ]; then
        docker tag ${image} $tag
    fi
}

# wait_docker_pull() - Function that waits for all docker pull processes
function wait_docker_pull {
    local counter=60
    local delay=${1:-60}

    sleep $delay
    while [ $(ps -ef | grep "docker pull" | wc -l) -gt 1 ]; do
        sleep $delay
        counter=$((counter - 1))
        if [ "$counter" -eq 0 ]; then
            break
        fi
    done
}

# run_docker_image() - Starts a Docker instance
function run_docker_image {
    install_docker
    docker run $@
}

# install_docker_compose() - Download and install docker-engine 
function install_docker_compose {
    local docker_compose_version=${1:-1.12.0}
    if [ ! -d /opt/docker ]; then
        mkdir /opt/docker
        curl -L https://github.com/docker/compose/releases/download/$docker_compose_version/docker-compose-`uname -s`-`uname -m` > /opt/docker/docker-compose
        chmod +x /opt/docker/docker-compose
    fi
}

# _install_ODL() - Download and Install OpenDayLight SDN controller
function _install_ODL {
    if [ ! -d /opt/opendaylight/current ]; then
        mkdir -p /opt/opendaylight/
        wget "https://nexus.opendaylight.org/content/repositories/public/org/opendaylight/integration/distribution-karaf/"$odl_version"/distribution-karaf-"$odl_version".tar.gz" -P /opt/
        tar xvf "/opt/distribution-karaf-"$odl_version".tar.gz" -C /tmp/
        mv "/tmp/distribution-karaf-"$odl_version /opt/opendaylight/current
        rm -rf "/opt/distribution-karaf-"$odl_version".tar.gz"
    fi
}

# start_ODL() - Start OpenDayLight SDN controller
function start_ODL {
    _install_ODL
    if [ -d /opt/opendaylight ]; then
        export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre
        /opt/opendaylight/current/bin/start
        sleep 180
        /opt/opendaylight/current/bin/client feature:install odl-dlux-all
    fi
}

# compile_src() - Function that compiles the java source code thru maven
function compile_src {
    local src_folder=$1
    pushd $src_folder
    local mvn_build='mvn clean install -DskipTests=true -Dmaven.test.skip=true -Dmaven.javadoc.skip=true -Dadditionalparam=-Xdoclint:none'
    if [[ "$debug" == "False" ]]; then
        mvn_build+=" -q"
    fi
    if [ -f pom.xml ]; then
        install_maven
        echo "Compiling $src_folder folder..."
        eval $mvn_build
    fi
    popd
}

# compile_repos() - Function that compiles source repositories for a given project
function compile_repos {
    local project=$1

    for repo in ${repos[$project]}; do
        compile_src ${src_folders[$project]}${repo#*$project}
    done
}

# build_docker_image() - Build Docker container image from source code
function build_docker_image {
    local src_folder=$1
    local profile=$2
    install_docker
    pushd $src_folder

    if [ -f pom.xml ]; then
        install_maven
        # Cleanup external repo
        sed -i 's|${docker.push.registry}/||g' pom.xml
        local docker_build="mvn clean package docker:build -DskipTests=true -Dmaven.test.skip=true -Dmaven.javadoc.skip=true"
        if [ $profile ]; then
            docker_build+=" -P $profile"
        fi
        if [[ "$debug" == "False" ]]; then
            docker_build+=" -q"
        fi
        if [ $http_proxy ]; then
            if ! grep -ql "docker.buildArg.http_proxy" pom.xml ; then
                docker_build+=" -Ddocker.buildArg.http_proxy=$http_proxy"
            fi
        if ! grep -ql "docker.buildArg.HTTP_PROXY" pom.xml ; then
            docker_build+=" -Ddocker.buildArg.HTTP_PROXY=$http_proxy"
        fi
        fi
        if [ $https_proxy ]; then
            if ! grep -ql "docker.buildArg.https_proxy" pom.xml ; then
                docker_build+=" -Ddocker.buildArg.https_proxy=$https_proxy"
            fi
            if ! grep -ql "docker.buildArg.HTTPS_PROXY" pom.xml ; then
                docker_build+=" -Ddocker.buildArg.HTTPS_PROXY=$https_proxy"
            fi
        fi
    elif [ -f Dockerfile ]; then
        # NOTE: Workaround for dmmapbc images
        sed -i '/LocalKey/d' Dockerfile
        sed -i "s/nexus3.onap.org\:10003\///g" Dockerfile
        local docker_build="docker build -t $profile -f ./Dockerfile ."
        if [ $http_proxy ]; then
            docker_build+=" --build-arg http_proxy=$http_proxy"
            docker_build+=" --build-arg HTTP_PROXY=$http_proxy"
        fi
        if [ $https_proxy ]; then
            docker_build+=" --build-arg https_proxy=$https_proxy"
            docker_build+=" --build-arg HTTPS_PROXY=$https_proxy"
        fi
    fi
    echo $docker_build
    eval $docker_build
    popd
}

# mount_external_partition() - Create partition and mount the external volume
function mount_external_partition {
    local dev_name="/dev/$1"
    local mount_dir=$2

    sfdisk $dev_name << EOF
;
EOF
    mkfs -t ext4 ${dev_name}1
    mkdir -p $mount_dir
    mount ${dev_name}1 $mount_dir
    echo "${dev_name}1  $mount_dir           ext4    errors=remount-ro,noatime,barrier=0 0       1" >> /etc/fstab
}

# add no_proxy values to environment, used for internal IPs generated at deploy time
function add_no_proxy_value {
    if [[ `grep "no_proxy" /etc/environment` ]]; then
        sed -i.bak "s/^no_proxy.*$/&,$1/" /etc/environment
    else
        echo "no_proxy=$1" >> /etc/environment
    fi
    if [[ `grep "NO_PROXY" /etc/environment` ]]; then
        sed -i.bak "s/^NO_PROXY.*$/&,$1/" /etc/environment
    else
        echo "NO_PROXY=$1" >> /etc/environment
    fi
}