summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSamuli Silvius <s.silvius@partner.samsung.com>2020-03-06 19:29:20 +0200
committerSamuli Silvius <s.silvius@partner.samsung.com>2020-03-09 16:13:51 +0200
commit523796ad09130868e8e31e942cd4d9712eb75934 (patch)
treea3ac00e533a442ab2d1a5f55da11ce64164dd8b2
parent367417cd7957916dffffef5d85e7cca304367c53 (diff)
Helper script for tailing POD logs
Simple way of utilizing kubectl to collect or tail POD logs. Supports tailing/collecting "kubectl logs" and any file content inside the POD container. Multiple PODs/containers and logs tailing/collecting started/collected with single command. POD and logs are configured in tail_config file. Issue-ID: INT-1260 Signed-off-by: Samuli Silvius <s.silvius@partner.samsung.com> Change-Id: I9e00e5f503fee2f86b75b1333973fd1569ae5aab
-rwxr-xr-xheat/vFW_CNF_CDS/templates/tools/tail_all.sh161
-rw-r--r--heat/vFW_CNF_CDS/templates/tools/tail_config17
2 files changed, 178 insertions, 0 deletions
diff --git a/heat/vFW_CNF_CDS/templates/tools/tail_all.sh b/heat/vFW_CNF_CDS/templates/tools/tail_all.sh
new file mode 100755
index 00000000..d18674ea
--- /dev/null
+++ b/heat/vFW_CNF_CDS/templates/tools/tail_all.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+# COPYRIGHT NOTICE STARTS HERE
+
+# Copyright 2020 . Samsung Electronics Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# COPYRIGHT NOTICE ENDS HERE
+
+set -e
+
+get_pod_name() {
+ kubectl -n ${NAMESPACE} get pod -l app="$1" -o jsonpath='{ .items[0].metadata.name }'
+}
+
+exec_in_pod() {
+ local pod="$1"
+ local container="$2"
+ shift 2
+ local cmd="$@"
+ kubectl --namespace ${NAMESPACE} exec ${pod} --container ${container} -- sh -c "${cmd}"
+}
+
+pod_logs() {
+ local pod="$1"
+ local container="$2"
+ local options="$3"
+ shift 3
+ kubectl --namespace ${NAMESPACE} logs ${pod} --container ${container} ${options:+"${options}"}
+}
+
+k8s_tail() {
+ local operation="$1"
+ local file_to_tail="$2"
+ local pod_label="$3"
+ local container="$4"
+ if [ -z ${container} ]; then
+ container=${pod_label}
+ fi
+ mkdir -p ${CACHE_FILE_DIR}
+ local pod=$(get_pod_name ${pod_label})
+ file_path_savable_form="$(sed 's#/#__#g' <<<"$file_to_tail")"
+ LINECOUNT_CACHE_FILE=${CACHE_FILE_DIR}/${pod}-${container}-${file_path_savable_form}
+ local line_count=0
+ if [ "${operation}" == "start" ]; then
+ if [ "${file_to_tail}" == "POD_LOG" ]; then
+ line_count=$(pod_logs ${pod} ${container} "" | wc -l)
+ else
+ if ! line_count=$(exec_in_pod ${pod} ${container} "wc -l ${file_to_tail}"); then
+ echo "Failed to get file ${file_to_tail} line count, maybe it does not exist. Using linecount 0." >&2
+ line_count=0
+ else
+ # parse out linecount from wc -l output
+ line_count=$(echo -e "${line_count}" | tail -1 | cut -d' ' -f1)
+ fi
+ fi
+ echo "${line_count}" > ${LINECOUNT_CACHE_FILE}
+ echo "Saved file ${file_to_tail} of POD ${pod_label} linecount ${line_count} into file ${LINECOUNT_CACHE_FILE}"
+ fi
+ if [ "${operation}" == "collect" ]; then
+ if [ ! -f ${LINECOUNT_CACHE_FILE} ]; then
+ echo "Linecount cache file ${LINECOUNT_CACHE_FILE} not found. Either tailing was not started or POD has restarted. Colected from log beginning." >&2
+ start_line=0
+ else
+ start_line=$(cat ${LINECOUNT_CACHE_FILE})
+ fi
+ echo "Tail pod's ${pod_label} container ${container} file ${file_to_tail} starting from line ${start_line} onwards" >&2
+ if [ "${file_to_tail}" == "POD_LOG" ]; then
+ current_line_count=$(pod_logs ${pod} ${container} "" | wc -l)
+ lines_from_end=$(( ${current_line_count}-${start_line} ))
+ echo "Execute kubectl logs for ${pod} POD's ${container} container. Log ${lines_from_end} lines from end"
+ pod_logs ${pod} ${container} --tail=${lines_from_end}
+ else
+ echo "Execute cmd "tail -n +${start_line} ${file_to_tail}" inside ${pod} POD's ${container} container"
+ exec_in_pod ${pod} ${container} "tail -n +${start_line} ${file_to_tail}"
+ fi
+ fi
+}
+
+tail_it() {
+ local operation=$1
+ for tail in "${TAILS[@]}"
+ do
+ tailarray=(${tail})
+ result=$(k8s_tail ${operation} ${tailarray[@]})
+ if [ "${operation}" == "collect" ]; then
+ mkdir -p ${RESULT_DIR}
+ log_file=$(basename ${tailarray[0]})
+ if [ "${log_file}" == "POD_LOG" ]; then
+ log_file=${log_file}.log
+ fi
+ pod=${tailarray[1]}
+ container="${pod:-tailarray[2]}"
+ out_file=${RESULT_DIR}/${pod}_${container}_${log_file}
+ echo -e "${result}" > ${out_file}
+ echo "Saved tail content to log file ${out_file}"
+ else
+ echo -e "${result}"
+ fi
+ done
+}
+
+
+##### MAIN #####
+if [ -f ./tail_config ]; then
+. ./tail_config
+fi
+NAMESPACE=${NAMESPACE:-onap}
+CACHE_FILE_DIR=${CACHE_FILE_DIR:-.k8s_tail}
+RESULT_DIR=${RESULT_DIR:-./tail_results}
+DEFAULT_TAILS=()
+TAILS=("${TAILS[@]:-${DEFAULT_TAILS[@]}}")
+
+case "$1" in
+ -h|--help|help|?|"")
+ echo "Script usage:"
+ echo "$0 start - Start pods' log tailing"
+ echo "$0 collect - Collect all logs currently tailed or get all logs as whole"
+ echo ""
+ echo "If start is not called before collect is called, collect gathers all logs completely. "
+ echo "Otherwise logs are collected only from the linecount of the time of start call."
+ echo ""
+ echo "Log files collected is configured with 'tail_config' file in the same directory with the script."
+ echo "Config file needs to fontain bash array variable named TAILS=() and format of each entry in array is"
+ echo " <file path> <POD's app label name> [<container name>]"
+ echo "where <file path> is actual file path inside the pod or special string 'POD_LOG' that means kubectl logs"
+ echo "where optional <container name> is to specify POD's container if many containers in the pod. By default same name as pod is used."
+ echo ""
+ echo 'Example: TAILS=('
+ echo ' "/app/logs/apih/metrics.log so"'
+ echo ' "/app/logs/bpmn/debug.log so-bpmn-infra"'
+ echo ' "/app/logs/vnfm-adapter/debug.log so-vnfm-adapter"'
+ echo ' "/var/log/onap/sdnc/karaf.log sdnc"'
+ echo ' "/tmp/app.out network-name-gen"'
+ echo ' "POD_LOG cds-blueprints-processor"'
+ echo ' "POD_LOG multicloud multicloud"'
+ echo ' "POD_LOG multicloud-k8s multicloud-k8s"'
+ echo ' "POD_LOG multicloud-k8s framework-artifactbroker"'
+ echo ' "/app/logs/openstack/error.log so-openstack-adapter"'
+ echo ' "/app/logs/openstack/debug.log so-openstack-adapter"'
+ echo ' )'
+ echo ""
+ ;;
+ start|collect)
+ tail_it $1
+ ;;
+ *)
+ echo "Wrong usage, check '$0 -h'" >&2
+ exit 1
+ ;;
+esac
diff --git a/heat/vFW_CNF_CDS/templates/tools/tail_config b/heat/vFW_CNF_CDS/templates/tools/tail_config
new file mode 100644
index 00000000..ca19623c
--- /dev/null
+++ b/heat/vFW_CNF_CDS/templates/tools/tail_config
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+NAMESPACE="onap"
+
+TAILS=(
+ "/app/logs/apih/metrics.log so"
+ "/app/logs/bpmn/debug.log so-bpmn-infra"
+ "/app/logs/vnfm-adapter/debug.log so-vnfm-adapter"
+ "/var/log/onap/sdnc/karaf.log sdnc"
+ "/tmp/app.out network-name-gen"
+ "POD_LOG cds-blueprints-processor"
+ "POD_LOG multicloud multicloud"
+ "POD_LOG multicloud-k8s multicloud-k8s"
+ "POD_LOG multicloud-k8s framework-artifactbroker"
+ "/app/logs/openstack/error.log so-openstack-adapter"
+ "/app/logs/openstack/debug.log so-openstack-adapter"
+)