diff options
Diffstat (limited to 'cdap3vm/config/cdap-config-template')
| -rw-r--r-- | cdap3vm/config/cdap-config-template/cdap-env.sh | 26 | ||||
| -rw-r--r-- | cdap3vm/config/cdap-config-template/cdap-site.xml | 61 | ||||
| -rwxr-xr-x | cdap3vm/config/cdap-config-template/common/common.sh | 309 | ||||
| -rw-r--r-- | cdap3vm/config/cdap-config-template/logback-container.xml | 70 | ||||
| -rw-r--r-- | cdap3vm/config/cdap-config-template/logback.xml | 81 |
5 files changed, 547 insertions, 0 deletions
diff --git a/cdap3vm/config/cdap-config-template/cdap-env.sh b/cdap3vm/config/cdap-config-template/cdap-env.sh new file mode 100644 index 0000000..eb962c9 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-env.sh @@ -0,0 +1,26 @@ +# Copyright © 2015 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Example environment variables. Please refer to the CDAP Administration Manual +# for more information. + +# If running CDAP on HDP 2.2+ the full HDP version string including iteration +# number must be passed in as an option. + +export JAVA_HOME=__JAVA_HOME__ +export OPTS="${OPTS} -Dhdp.version=__HDP_VERSION__" + +PATH=$PATH:__NODEJS_BIN__ +# Ensure SPARK_HOME is set for Spark support. +# export SPARK_HOME="/usr/lib/spark" diff --git a/cdap3vm/config/cdap-config-template/cdap-site.xml b/cdap3vm/config/cdap-config-template/cdap-site.xml new file mode 100644 index 0000000..c0eabda --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-site.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Copyright © 2014 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> +<configuration> + <!-- + Your site level configuration goes here + --> +<property> + <name>zookeeper.quorum</name> + <value>__HDP_ZOOKEEPER_QUORUM__/${root.namespace}</value> + <description> + ZooKeeper quorum string; specifies the ZooKeeper host:port; + substitute the quorum for the components shown here (FQDN1:2181,FQDN2:2181) + </description> +</property> +<property> + <name>router.server.address</name> + <value>__CDAP_ROUTER_HOST__</value> + <description>CDAP Router address to which CDAP UI connects</description> +</property> +<property> + <name>explore.enabled</name> + <value>false</value> + <description>Enable Explore functionality</description> +</property> +<property> + <name>enable.unrecoverable.reset</name> + <value>true</value> + <description>Needed for allowing deletion of namespaces</description> +</property> +<property> + <name>app.program.jvm.opts</name> + <value>-XX:MaxPermSize=128M ${twill.jvm.gc.opts} -Dhdp.version=__HDP_VERSION__ -Dspark.yarn.am.extraJavaOptions=-Dhdp.version=__HDP_VERSION__</value> + <description>Java options for all program containers</description> +</property> + +<property> + <name>kafka.seed.brokers</name> + <value>__CDAP_KAFKA_HOST__:9092</value> + <description> + Comma-separated list of CDAP Kafka service brokers; for distributed CDAP, + replace with list of FQDN:port brokers + </description> +</property> + + +</configuration> diff --git a/cdap3vm/config/cdap-config-template/common/common.sh b/cdap3vm/config/cdap-config-template/common/common.sh new file mode 100755 index 0000000..c58fb6a --- /dev/null +++ b/cdap3vm/config/cdap-config-template/common/common.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash + +# +# Copyright © 2015-2016 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# checks if there exists a PID that is already running. return 0 idempotently + +export JAVA_HOME=__JAVA_HOME__ +PATH=$PATH:__NODEJS_BIN__ + +cdap_check_before_start() { + if [ -f ${pid} ]; then + if kill -0 $(<${pid}) > /dev/null 2>&1; then + echo "${APP} running as process $(<${pid}). Stop it first." + return 1 + fi + fi + return 0 +} + +cdap_create_pid_dir() { + mkdir -p "${PID_DIR}" +} + +die() { + echo "ERROR: ${*}" + exit 1 +} + +# usage: cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true +cdap_get_conf() { + local __pn=${1} __fn=${2} __default=${3} __result= + # Check for xmllint + [[ $(which xmllint 2>/dev/null) ]] || { + case ${PLATFORM} in + RHEL) die "Cannot locate xmllint, is libxml2 installed?" ;; + UBUNTU) die "Cannot locate xmllint, is libxml2-utils installed?" ;; + esac + # If we get here, die + die "Cannot locate xmllint, are XML tools installed?" + } + # Get property from file, return last result, if multiple are returned + __property="cat //configuration/property[name='${__pn}']/value[text()]" + __sed_fu='/^\//d;s/^.*<value>//;s/<\/value>.*$//' + __result=$(echo "${__property}" | xmllint --shell "${__fn}" | sed "${__sed_fu}" | tail -n 1) + # Found result, echo it and return 0 + [[ -n "${__result}" ]] && echo ${__result} && return 0 + # No result, echo default and return 0 + [[ -n "${__default}" ]] && echo ${__default} && return 0 + return 1 +} + +# Rotates the basic start/stop logs +cdap_rotate_log () { + local log=${1} num=5 prev=0 + [[ -n "${2}" ]] && num=${2} + if [ -f "${log}" ]; then # rotate logs + while [ ${num} -gt 1 ]; do + prev=$((${num} - 1)) + [ -f "${log}.${prev}" ] && mv -f "${log}.${prev}" "${log}.${num}" + num=${prev} + done + mv -f "${log}" "${log}.${num}" + fi +} + +# CDAP kinit using properties from cdap-site.xml +cdap_kinit() { + local __principal=${CDAP_PRINCIPAL:-$(cdap_get_conf "cdap.master.kerberos.principal" "${CDAP_CONF}"/cdap-site.xml)} + local __keytab=${CDAP_KEYTAB:-$(cdap_get_conf "cdap.master.kerberos.keytab" "${CDAP_CONF}"/cdap-site.xml)} + if [ -z "${__principal}" -o -z "${__keytab}" ]; then + echo "ERROR: Both cdap.master.kerberos.principal and cdap.master.kerberos.keytab must be configured for Kerberos-enabled clusters!" + return 1 + fi + if [ ! -r "${__keytab}" ]; then + echo "ERROR: Cannot read keytab: ${__keytab}" + return 1 + fi + if [[ $(which kinit 2>/dev/null) ]]; then + # Replace _HOST in principal w/ FQDN, like Hadoop does + kinit -kt "${__keytab}" "${__principal/_HOST/`hostname -f`}" + if [[ ! $? ]]; then + echo "ERROR: Failed executing 'kinit -kt \"${__keytab}\" \"${__principal/_HOST/`hostname -f`}\"'" + return 1 + fi + else + echo "ERROR: Cannot locate kinit! Please, ensure the appropriate Kerberos utilities are installed" + return 1 + fi + return 0 +} + +# Attempts to find JAVA in few ways. This is used by UI's config-tool, so verify that changes here do not break that +cdap_set_java () { + # Determine the Java command to use to start the JVM. + if [ -n "${JAVA_HOME}" ] ; then + if [ -x "${JAVA_HOME}/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + export JAVA="${JAVA_HOME}/jre/sh/java" + else + export JAVA="${JAVA_HOME}/bin/java" + fi + if [ ! -x "${JAVA}" ] ; then + echo "ERROR: JAVA_HOME is set to an invalid directory: ${JAVA_HOME} + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + else + export JAVA="java" + which java >/dev/null 2>&1 && return 0 + # If we get here, we've failed this city + echo "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + return 0 +} + +# Sets the correct HBase support library to use, based on what version exists in the classpath +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hbase() { + # Why is this here? Is this not redundant? + if [ -z "${JAVA}" ]; then + echo "ERROR: JAVA is not yet set, cannot determine HBase version" + return 1 + fi + + retvalue=1 + if [ -z "${HBASE_VERSION}" ]; then + HBASE_VERSION=$(${JAVA} -cp ${CLASSPATH} co.cask.cdap.data2.util.hbase.HBaseVersion 2> /dev/null) + retvalue=$? + else + retvalue=0 + fi + + # only set HBase version if previous call succeeded (may fail for components that don't use HBase) + if [ ${retvalue} == 0 ]; then + case ${HBASE_VERSION} in + 0.96*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.96/lib/*" + ;; + 0.98*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.98/lib/*" + ;; + 1.0-cdh5.5* | 1.0-cdh5.6*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh5.5.0/lib/*" + ;; + 1.0-cdh*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh/lib/*" + ;; + 1.0*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0/lib/*" + ;; + 1.1*) + hbasecompat="$CDAP_HOME/hbase-compat-1.1/lib/*" + ;; + *) + echo "ERROR: Unknown/unsupported version of HBase found: ${HBASE_VERSION}" + return 1 + ;; + esac + if [ -n "${hbasecompat}" ]; then + CLASSPATH="${hbasecompat}:${CLASSPATH}" + else + # When will we ever hit this case? + echo "ERROR: Failed to find installed hbase-compat jar for version ${HBASE_VERSION}." + echo " Is the hbase-compat-* package installed?" + return 1 + fi + fi + export CLASSPATH + return 0 +} + +# set the classpath to include hadoop and hbase dependencies +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_classpath() { + COMP_HOME=${1} + CCONF=${2} + if [ -n "${HBASE_HOME}" ]; then + HBASE_CP=$(${HBASE_HOME}/bin/hbase classpath) + elif [[ $(which hbase 2>/dev/null) ]]; then + HBASE_CP=$(hbase classpath) + fi + + # Where is this used outside this function? + export HBASE_CP + + # In order to ensure that we can do hacks, need to make sure classpath is sorted + # so that cdap jars are placed earlier in the classpath than twill or hadoop jars + COMP_LIB=$(find -L "${COMP_HOME}/lib" -type f | sort | tr '\n' ':') + + if [ -n "${HBASE_CP}" ]; then + CP="${COMP_LIB}:${HBASE_CP}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + else + # assume Hadoop/HBase libs are included via EXTRA_CLASSPATH + echo "WARN: could not find Hadoop and HBase libraries" + CP="${COMP_LIB}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + fi + + # Setup classpaths. + if [ -n "${CLASSPATH}" ]; then + CLASSPATH="${CLASSPATH}:${CP}" + else + CLASSPATH="${CP}" + fi + + export CLASSPATH +} + +# Determine Hive classpath, and set EXPLORE_CLASSPATH. +# Hive classpath is not added as part of system classpath as hive jars bundle unrelated jars like guava, +# and hence need to be isolated. +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hive_classpath() { + local __explore=${EXPLORE_ENABLED:-$(cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true)} + if [[ "${__explore}" == "true" ]]; then + if [ -z "${HIVE_HOME}" -o -z "${HIVE_CONF_DIR}" -o -z "${HADOOP_CONF_DIR}" ]; then + __secure=${KERBEROS_ENABLED:-$(cdap_get_conf "kerberos.auth.enabled" "${CDAP_CONF}"/cdap-site.xml false)} + if [[ "${__secure}" == "true" ]]; then + cdap_kinit || return 1 + fi + + if [[ $(which hive 2>/dev/null) ]]; then + ERR_FILE=$(mktemp) + HIVE_VAR_OUT=$(hive -e 'set -v' 2>${ERR_FILE}) + __ret=$? + HIVE_ERR_MSG=$(< ${ERR_FILE}) + rm ${ERR_FILE} + if [ ${__ret} -ne 0 ]; then + echo "ERROR - While determining Hive classpath, failed to get Hive settings using: hive -e 'set -v'" + echo "If you do not want run CDAP with Hive functionality, set the 'explore.enabled' property in cdap-site.xml to 'false'" + echo "Otherwise, check that the Hive client is installed, and that Hive and HDFS are running." + echo "stderr:" + echo "${HIVE_ERR_MSG}" + return 1 + fi + HIVE_VARS=$(echo ${HIVE_VAR_OUT} | tr ' ' '\n') + # Quotes preserve whitespace + HIVE_HOME=${HIVE_HOME:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_HOME=' | cut -d= -f2)} + HIVE_CONF_DIR=${HIVE_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_CONF_DIR=' | cut -d= -f2)} + HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HADOOP_CONF_DIR=' | cut -d= -f2)} + fi + fi + + # If Hive classpath is successfully determined, derive explore + # classpath from it and export it to use it in the launch command + if [ -n "${HIVE_HOME}" -a -n "${HIVE_CONF_DIR}" -a -n "${HADOOP_CONF_DIR}" ]; then + EXPLORE_CONF_FILES=$(ls -1dF ${HIVE_CONF_DIR}/* ${HADOOP_CONF_DIR}/* | sed -e '/\/$/d' | tr '\n' ':') + EXPLORE_CLASSPATH=$(ls -1 ${HIVE_HOME}/lib/hive-exec-* ${HIVE_HOME}/lib/*.jar | tr '\n' ':') + export EXPLORE_CONF_FILES EXPLORE_CLASSPATH + fi + fi +} + +# Check that directory /var/tmp/cdap exists in the master node, or create it +cdap_check_or_create_master_local_dir() { + mkdir -p "${LOCAL_DIR}" +} + +# check and set classpath if in development enviroment +cdap_check_and_set_classpath_for_dev_environment () { + APP_HOME=${1} + + # Detect if we are in development. + IN_DEV_ENVIRONMENT=${IN_DEV_ENVIRONMENT:-false} + + # for developers only, add flow and flow related stuff to class path. + if [[ "${IN_DEV_ENVIRONMENT}" == "true" ]]; then + echo "Constructing classpath for development environment ..." + [[ -f "${APP_HOME}"/build/generated-classpath ]] && CLASSPATH+=":$(<${APP_HOME}/build/generated-classpath)" + [[ -d "${APP_HOME}"/build/classes ]] && CLASSPATH+=":${APP_HOME}/build/classes/main:${APP_HOME}/conf/*" + [[ -d "${APP_HOME}"/../data-fabric/build/classes ]] && CLASSPATH+=":${APP_HOME}/../data-fabric/build/classes/main" + [[ -d "${APP_HOME}"/../common/build/classes ]] && CLASSPATH+=":${APP_HOME}/../common/build/classes/main" + [[ -d "${APP_HOME}"/../gateway/build/classes ]] && CLASSPATH+=":${APP_HOME}/../gateway/build/classes/main" + export CLASSPATH + fi +} + +HOSTNAME=$(hostname -f) +export LOG_PREFIX=${APP}-${IDENT_STRING}-${HOSTNAME} +export LOGFILE=${LOG_PREFIX}.log +loglog="${LOG_DIR}/${LOGFILE}" + +pid=${PID_DIR}/${APP}-${IDENT_STRING}.pid +loggc="${LOG_DIR}/${LOG_PREFIX}.gc" + +export NICENESS=${NICENESS:-0} diff --git a/cdap3vm/config/cdap-config-template/logback-container.xml b/cdap3vm/config/cdap-config-template/logback-container.xml new file mode 100644 index 0000000..f1957d8 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback-container.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Copyright © 2015 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> + +<!-- + Logback used by containers that run in YARN eg. Flows, MR, Explore +--> +<configuration> + + <!-- + Disabling some chatty loggers. + --> + <logger name="org.apache.commons.beanutils" level="ERROR"/> + <logger name="org.apache.zookeeper.server" level="ERROR"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="com.ning" level="WARN"/> + <logger name="org.apache.spark" level="WARN"/> + <logger name="org.spark-project" level="WARN"/> + <logger name="org.apache.hadoop" level="WARN"/> + <logger name="org.apache.hive" level="WARN"/> + <logger name="org.quartz.core" level="WARN"/> + <logger name="org.eclipse.jetty" level="WARN"/> + <logger name="io.netty.util.internal" level="WARN"/> + + <logger name="org.apache.twill" level="WARN"/> + <logger name="co.cask.cdap" level="INFO"/> + + <!-- Redirected stdout and stderr of Hive operations --> + <logger name="Explore.stdout" level="INFO"/> + <logger name="Explore.stderr" level="INFO"/> + + + <!-- quick workaround suggested by CASK for ticket #666 [DE257314] --> + <logger name="org.apache.hadoop.io.retry.RetryInvocationHandler" level="ERROR"/> + + <appender name="Rolling" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- LOG_DIRS is the environment variable set by YARN for container logs --> + <file>${LOG_DIRS}/program.log</file> + <encoder> + <pattern>%d{ISO8601} - %-5p [%t:%logger{1}@%L] - %m%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- Daily rollover at midnight--> + <fileNamePattern>${LOG_DIRS}/program.%d.log</fileNamePattern> + + <!-- Keep 2 weeks of history --> + <maxHistory>14</maxHistory> + </rollingPolicy> + </appender> + + <root level="INFO"> + <appender-ref ref="Rolling"/> + </root> + +</configuration> + diff --git a/cdap3vm/config/cdap-config-template/logback.xml b/cdap3vm/config/cdap-config-template/logback.xml new file mode 100644 index 0000000..e036683 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Copyright © 2014 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> + +<configuration> + + <!-- + Disabling some chatty loggers. + --> + <logger name="org.apache.commons.beanutils" level="ERROR"/> + <logger name="org.apache.zookeeper.server" level="ERROR"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="com.ning" level="WARN"/> + <logger name="org.apache.spark" level="WARN"/> + <logger name="org.apache.hadoop" level="WARN"/> + <logger name="org.apache.hive" level="WARN"/> + <logger name="org.quartz.core" level="WARN"/> + <logger name="org.eclipse.jetty" level="WARN"/> + <logger name="org.apache.twill" level="WARN"/> + <logger name="co.cask.cdap" level="INFO"/> + + <appender name="Console" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern> + </encoder> + </appender> + + <root level="INFO"> + <appender-ref ref="Console"/> + </root> + + <!-- + Uncomment the following section to enable access logging by the Authentication Server and Router processes. + Log output will be in the standard Apache HTTPd access log format (see + http://httpd.apache.org/docs/2.2/logs.html#accesslog). + --> + <!-- + <appender name="AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>access.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>access.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <logger name="http-access" level="TRACE" additivity="false"> + <appender-ref ref="AUDIT" /> + </logger> + <appender name="EXTERNAL_AUTH_AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>external_auth_access.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>external_auth_access.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <logger name="external-auth-access" level="TRACE" additivity="false"> + <appender-ref ref="EXTERNAL_AUTH_AUDIT" /> + </logger> + --> + +</configuration> + |