diff options
Diffstat (limited to 'cdap3vm/config/cdap-config-template')
| -rw-r--r-- | cdap3vm/config/cdap-config-template/cdap-site.xml | 18 | ||||
| -rwxr-xr-x | cdap3vm/config/cdap-config-template/common/common.sh | 60 | ||||
| -rw-r--r-- | cdap3vm/config/cdap-config-template/logback-container.xml | 17 |
3 files changed, 80 insertions, 15 deletions
diff --git a/cdap3vm/config/cdap-config-template/cdap-site.xml b/cdap3vm/config/cdap-config-template/cdap-site.xml index c0eabda..1ce75bd 100644 --- a/cdap3vm/config/cdap-config-template/cdap-site.xml +++ b/cdap3vm/config/cdap-config-template/cdap-site.xml @@ -57,5 +57,23 @@ </description> </property> +<!-- [171660] Tuning parameter changes for log saver and master memory --> +<property> + <name>log.saver.run.memory.megs</name> + <value>1024</value> + <description>Memory in megabytes allocated for log saver instances to run in YARN</description> +</property> + +<property> + <name>log.saver.num.instances</name> + <value>2</value> + <description>Number of log saver instances to run in YARN</description> +</property> + +<property> + <name>master.service.memory.mb</name> + <value>2048</value> + <description>Memory in megabytes for each master service instance</description> +</property> </configuration> diff --git a/cdap3vm/config/cdap-config-template/common/common.sh b/cdap3vm/config/cdap-config-template/common/common.sh index c58fb6a..394602e 100755 --- a/cdap3vm/config/cdap-config-template/common/common.sh +++ b/cdap3vm/config/cdap-config-template/common/common.sh @@ -15,11 +15,10 @@ # License for the specific language governing permissions and limitations under # the License. -# checks if there exists a PID that is already running. return 0 idempotently - export JAVA_HOME=__JAVA_HOME__ PATH=$PATH:__NODEJS_BIN__ +# checks if there exists a PID that is already running. return 0 idempotently cdap_check_before_start() { if [ -f ${pid} ]; then if kill -0 $(<${pid}) > /dev/null 2>&1; then @@ -169,7 +168,10 @@ cdap_set_hbase() { hbasecompat="${CDAP_HOME}/hbase-compat-1.0/lib/*" ;; 1.1*) - hbasecompat="$CDAP_HOME/hbase-compat-1.1/lib/*" + hbasecompat="${CDAP_HOME}/hbase-compat-1.1/lib/*" + ;; + 1.2-cdh*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.2-cdh5.7.0/lib/*" ;; *) echo "ERROR: Unknown/unsupported version of HBase found: ${HBASE_VERSION}" @@ -207,7 +209,7 @@ cdap_set_classpath() { # In order to ensure that we can do hacks, need to make sure classpath is sorted # so that cdap jars are placed earlier in the classpath than twill or hadoop jars - COMP_LIB=$(find -L "${COMP_HOME}/lib" -type f | sort | tr '\n' ':') + COMP_LIB=$(find -L "${COMP_HOME}/lib" -type f 2>/dev/null | sort | tr '\n' ':') if [ -n "${HBASE_CP}" ]; then CP="${COMP_LIB}:${HBASE_CP}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" @@ -242,9 +244,16 @@ cdap_set_hive_classpath() { cdap_kinit || return 1 fi - if [[ $(which hive 2>/dev/null) ]]; then + # Use ${HIVE_HOME} if set + if [ -n "${HIVE_HOME}" ]; then + HIVE_CMD=${HIVE_HOME}/bin/hive + else + HIVE_CMD=hive + fi + + if [[ $(which ${HIVE_CMD} 2>/dev/null) ]]; then ERR_FILE=$(mktemp) - HIVE_VAR_OUT=$(hive -e 'set -v' 2>${ERR_FILE}) + HIVE_VAR_OUT=$(${HIVE_CMD} -e 'set -v' 2>${ERR_FILE}) __ret=$? HIVE_ERR_MSG=$(< ${ERR_FILE}) rm ${ERR_FILE} @@ -261,6 +270,7 @@ cdap_set_hive_classpath() { HIVE_HOME=${HIVE_HOME:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_HOME=' | cut -d= -f2)} HIVE_CONF_DIR=${HIVE_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_CONF_DIR=' | cut -d= -f2)} HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HADOOP_CONF_DIR=' | cut -d= -f2)} + HIVE_EXEC_ENGINE=${HIVE_EXEC_ENGINE:-$(echo -e "${HIVE_VARS}" | grep '^hive.execution.engine=' | cut -d= -f2)} fi fi @@ -269,11 +279,49 @@ cdap_set_hive_classpath() { if [ -n "${HIVE_HOME}" -a -n "${HIVE_CONF_DIR}" -a -n "${HADOOP_CONF_DIR}" ]; then EXPLORE_CONF_FILES=$(ls -1dF ${HIVE_CONF_DIR}/* ${HADOOP_CONF_DIR}/* | sed -e '/\/$/d' | tr '\n' ':') EXPLORE_CLASSPATH=$(ls -1 ${HIVE_HOME}/lib/hive-exec-* ${HIVE_HOME}/lib/*.jar | tr '\n' ':') + if [ -n "${TEZ_HOME}" -a -n "${TEZ_CONF_DIR}" ]; then + # tez-site.xml also need to be passed to explore service + EXPLORE_CONF_FILES=${EXPLORE_CONF_FILES}:${TEZ_CONF_DIR}/tez-site.xml: + fi + if [[ "${HIVE_EXEC_ENGINE}" == "spark" ]]; then + # We require SPARK_HOME to be set for CDAP to include the Spark assembly JAR for Explore + cdap_set_spark || die "Unable to get SPARK_HOME, but default Hive engine is Spark" + fi export EXPLORE_CONF_FILES EXPLORE_CLASSPATH fi fi } +# Get SPARK_HOME +cdap_set_spark() { + local readonly __saved_stty=$(stty -g 2>/dev/null) + # First, see if we're set to something sane + if [ -n "${SPARK_HOME}" -a -d "${SPARK_HOME}" ]; then + export SPARK_HOME + return 0 # SPARK_HOME is set, already + else + if [[ $(which spark-shell 2>/dev/null) ]]; then + ERR_FILE=$(mktemp) + SPARK_VAR_OUT=$(echo 'for ((key, value) <- sys.env) println (key + "=" + value); exit' | spark-shell --master local 2>${ERR_FILE}) + __ret=$? + # spark-shell invocation above does not properly restore the stty. + stty ${__saved_stty} + SPARK_ERR_MSG=$(< ${ERR_FILE}) + rm ${ERR_FILE} + if [ ${__ret} -ne 0 ]; then + echo "ERROR - While determining Spark home, failed to get Spark settings using: spark-shell --master local" + echo "stderr:" + echo "${SPARK_ERR_MSG}" + return 1 + fi + SPARK_HOME=$(echo -e "${SPARK_VAR_OUT}" | grep ^SPARK_HOME= | cut -d= -f2) + export SPARK_HOME + return 0 + fi + return 1 + fi +} + # Check that directory /var/tmp/cdap exists in the master node, or create it cdap_check_or_create_master_local_dir() { mkdir -p "${LOCAL_DIR}" diff --git a/cdap3vm/config/cdap-config-template/logback-container.xml b/cdap3vm/config/cdap-config-template/logback-container.xml index f1957d8..c693ace 100644 --- a/cdap3vm/config/cdap-config-template/logback-container.xml +++ b/cdap3vm/config/cdap-config-template/logback-container.xml @@ -2,13 +2,13 @@ <!-- Copyright © 2015 Cask Data, Inc. - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -43,19 +43,18 @@ <logger name="Explore.stdout" level="INFO"/> <logger name="Explore.stderr" level="INFO"/> - - <!-- quick workaround suggested by CASK for ticket #666 [DE257314] --> - <logger name="org.apache.hadoop.io.retry.RetryInvocationHandler" level="ERROR"/> + <!-- quick workaround suggested by CASK for ticket #666 [DE257314] --> + <logger name="org.apache.hadoop.io.retry.RetryInvocationHandler" level="ERROR"/> <appender name="Rolling" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <!-- LOG_DIRS is the environment variable set by YARN for container logs --> - <file>${LOG_DIRS}/program.log</file> + <!-- CDAP_LOG_DIR is the environment variable set by CDAP for logs --> + <file>${CDAP_LOG_DIR}/program.log</file> <encoder> <pattern>%d{ISO8601} - %-5p [%t:%logger{1}@%L] - %m%n</pattern> </encoder> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <!-- Daily rollover at midnight--> - <fileNamePattern>${LOG_DIRS}/program.%d.log</fileNamePattern> + <fileNamePattern>${CDAP_LOG_DIR}/program.%d.log</fileNamePattern> <!-- Keep 2 weeks of history --> <maxHistory>14</maxHistory> |