diff options
Diffstat (limited to 'cdap3vm')
129 files changed, 8326 insertions, 0 deletions
diff --git a/cdap3vm/config/cdap-config-template/cdap-env.sh b/cdap3vm/config/cdap-config-template/cdap-env.sh new file mode 100644 index 0000000..eb962c9 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-env.sh @@ -0,0 +1,26 @@ +# Copyright © 2015 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Example environment variables. Please refer to the CDAP Administration Manual +# for more information. + +# If running CDAP on HDP 2.2+ the full HDP version string including iteration +# number must be passed in as an option. + +export JAVA_HOME=__JAVA_HOME__ +export OPTS="${OPTS} -Dhdp.version=__HDP_VERSION__" + +PATH=$PATH:__NODEJS_BIN__ +# Ensure SPARK_HOME is set for Spark support. +# export SPARK_HOME="/usr/lib/spark" diff --git a/cdap3vm/config/cdap-config-template/cdap-site.xml b/cdap3vm/config/cdap-config-template/cdap-site.xml new file mode 100644 index 0000000..c0eabda --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-site.xml @@ -0,0 +1,61 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Copyright © 2014 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> +<configuration> + <!-- + Your site level configuration goes here + --> +<property> + <name>zookeeper.quorum</name> + <value>__HDP_ZOOKEEPER_QUORUM__/${root.namespace}</value> + <description> + ZooKeeper quorum string; specifies the ZooKeeper host:port; + substitute the quorum for the components shown here (FQDN1:2181,FQDN2:2181) + </description> +</property> +<property> + <name>router.server.address</name> + <value>__CDAP_ROUTER_HOST__</value> + <description>CDAP Router address to which CDAP UI connects</description> +</property> +<property> + <name>explore.enabled</name> + <value>false</value> + <description>Enable Explore functionality</description> +</property> +<property> + <name>enable.unrecoverable.reset</name> + <value>true</value> + <description>Needed for allowing deletion of namespaces</description> +</property> +<property> + <name>app.program.jvm.opts</name> + <value>-XX:MaxPermSize=128M ${twill.jvm.gc.opts} -Dhdp.version=__HDP_VERSION__ -Dspark.yarn.am.extraJavaOptions=-Dhdp.version=__HDP_VERSION__</value> + <description>Java options for all program containers</description> +</property> + +<property> + <name>kafka.seed.brokers</name> + <value>__CDAP_KAFKA_HOST__:9092</value> + <description> + Comma-separated list of CDAP Kafka service brokers; for distributed CDAP, + replace with list of FQDN:port brokers + </description> +</property> + + +</configuration> diff --git a/cdap3vm/config/cdap-config-template/common/common.sh b/cdap3vm/config/cdap-config-template/common/common.sh new file mode 100755 index 0000000..c58fb6a --- /dev/null +++ b/cdap3vm/config/cdap-config-template/common/common.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash + +# +# Copyright © 2015-2016 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# checks if there exists a PID that is already running. return 0 idempotently + +export JAVA_HOME=__JAVA_HOME__ +PATH=$PATH:__NODEJS_BIN__ + +cdap_check_before_start() { + if [ -f ${pid} ]; then + if kill -0 $(<${pid}) > /dev/null 2>&1; then + echo "${APP} running as process $(<${pid}). Stop it first." + return 1 + fi + fi + return 0 +} + +cdap_create_pid_dir() { + mkdir -p "${PID_DIR}" +} + +die() { + echo "ERROR: ${*}" + exit 1 +} + +# usage: cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true +cdap_get_conf() { + local __pn=${1} __fn=${2} __default=${3} __result= + # Check for xmllint + [[ $(which xmllint 2>/dev/null) ]] || { + case ${PLATFORM} in + RHEL) die "Cannot locate xmllint, is libxml2 installed?" ;; + UBUNTU) die "Cannot locate xmllint, is libxml2-utils installed?" ;; + esac + # If we get here, die + die "Cannot locate xmllint, are XML tools installed?" + } + # Get property from file, return last result, if multiple are returned + __property="cat //configuration/property[name='${__pn}']/value[text()]" + __sed_fu='/^\//d;s/^.*<value>//;s/<\/value>.*$//' + __result=$(echo "${__property}" | xmllint --shell "${__fn}" | sed "${__sed_fu}" | tail -n 1) + # Found result, echo it and return 0 + [[ -n "${__result}" ]] && echo ${__result} && return 0 + # No result, echo default and return 0 + [[ -n "${__default}" ]] && echo ${__default} && return 0 + return 1 +} + +# Rotates the basic start/stop logs +cdap_rotate_log () { + local log=${1} num=5 prev=0 + [[ -n "${2}" ]] && num=${2} + if [ -f "${log}" ]; then # rotate logs + while [ ${num} -gt 1 ]; do + prev=$((${num} - 1)) + [ -f "${log}.${prev}" ] && mv -f "${log}.${prev}" "${log}.${num}" + num=${prev} + done + mv -f "${log}" "${log}.${num}" + fi +} + +# CDAP kinit using properties from cdap-site.xml +cdap_kinit() { + local __principal=${CDAP_PRINCIPAL:-$(cdap_get_conf "cdap.master.kerberos.principal" "${CDAP_CONF}"/cdap-site.xml)} + local __keytab=${CDAP_KEYTAB:-$(cdap_get_conf "cdap.master.kerberos.keytab" "${CDAP_CONF}"/cdap-site.xml)} + if [ -z "${__principal}" -o -z "${__keytab}" ]; then + echo "ERROR: Both cdap.master.kerberos.principal and cdap.master.kerberos.keytab must be configured for Kerberos-enabled clusters!" + return 1 + fi + if [ ! -r "${__keytab}" ]; then + echo "ERROR: Cannot read keytab: ${__keytab}" + return 1 + fi + if [[ $(which kinit 2>/dev/null) ]]; then + # Replace _HOST in principal w/ FQDN, like Hadoop does + kinit -kt "${__keytab}" "${__principal/_HOST/`hostname -f`}" + if [[ ! $? ]]; then + echo "ERROR: Failed executing 'kinit -kt \"${__keytab}\" \"${__principal/_HOST/`hostname -f`}\"'" + return 1 + fi + else + echo "ERROR: Cannot locate kinit! Please, ensure the appropriate Kerberos utilities are installed" + return 1 + fi + return 0 +} + +# Attempts to find JAVA in few ways. This is used by UI's config-tool, so verify that changes here do not break that +cdap_set_java () { + # Determine the Java command to use to start the JVM. + if [ -n "${JAVA_HOME}" ] ; then + if [ -x "${JAVA_HOME}/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + export JAVA="${JAVA_HOME}/jre/sh/java" + else + export JAVA="${JAVA_HOME}/bin/java" + fi + if [ ! -x "${JAVA}" ] ; then + echo "ERROR: JAVA_HOME is set to an invalid directory: ${JAVA_HOME} + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + else + export JAVA="java" + which java >/dev/null 2>&1 && return 0 + # If we get here, we've failed this city + echo "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + return 0 +} + +# Sets the correct HBase support library to use, based on what version exists in the classpath +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hbase() { + # Why is this here? Is this not redundant? + if [ -z "${JAVA}" ]; then + echo "ERROR: JAVA is not yet set, cannot determine HBase version" + return 1 + fi + + retvalue=1 + if [ -z "${HBASE_VERSION}" ]; then + HBASE_VERSION=$(${JAVA} -cp ${CLASSPATH} co.cask.cdap.data2.util.hbase.HBaseVersion 2> /dev/null) + retvalue=$? + else + retvalue=0 + fi + + # only set HBase version if previous call succeeded (may fail for components that don't use HBase) + if [ ${retvalue} == 0 ]; then + case ${HBASE_VERSION} in + 0.96*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.96/lib/*" + ;; + 0.98*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.98/lib/*" + ;; + 1.0-cdh5.5* | 1.0-cdh5.6*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh5.5.0/lib/*" + ;; + 1.0-cdh*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh/lib/*" + ;; + 1.0*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0/lib/*" + ;; + 1.1*) + hbasecompat="$CDAP_HOME/hbase-compat-1.1/lib/*" + ;; + *) + echo "ERROR: Unknown/unsupported version of HBase found: ${HBASE_VERSION}" + return 1 + ;; + esac + if [ -n "${hbasecompat}" ]; then + CLASSPATH="${hbasecompat}:${CLASSPATH}" + else + # When will we ever hit this case? + echo "ERROR: Failed to find installed hbase-compat jar for version ${HBASE_VERSION}." + echo " Is the hbase-compat-* package installed?" + return 1 + fi + fi + export CLASSPATH + return 0 +} + +# set the classpath to include hadoop and hbase dependencies +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_classpath() { + COMP_HOME=${1} + CCONF=${2} + if [ -n "${HBASE_HOME}" ]; then + HBASE_CP=$(${HBASE_HOME}/bin/hbase classpath) + elif [[ $(which hbase 2>/dev/null) ]]; then + HBASE_CP=$(hbase classpath) + fi + + # Where is this used outside this function? + export HBASE_CP + + # In order to ensure that we can do hacks, need to make sure classpath is sorted + # so that cdap jars are placed earlier in the classpath than twill or hadoop jars + COMP_LIB=$(find -L "${COMP_HOME}/lib" -type f | sort | tr '\n' ':') + + if [ -n "${HBASE_CP}" ]; then + CP="${COMP_LIB}:${HBASE_CP}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + else + # assume Hadoop/HBase libs are included via EXTRA_CLASSPATH + echo "WARN: could not find Hadoop and HBase libraries" + CP="${COMP_LIB}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + fi + + # Setup classpaths. + if [ -n "${CLASSPATH}" ]; then + CLASSPATH="${CLASSPATH}:${CP}" + else + CLASSPATH="${CP}" + fi + + export CLASSPATH +} + +# Determine Hive classpath, and set EXPLORE_CLASSPATH. +# Hive classpath is not added as part of system classpath as hive jars bundle unrelated jars like guava, +# and hence need to be isolated. +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hive_classpath() { + local __explore=${EXPLORE_ENABLED:-$(cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true)} + if [[ "${__explore}" == "true" ]]; then + if [ -z "${HIVE_HOME}" -o -z "${HIVE_CONF_DIR}" -o -z "${HADOOP_CONF_DIR}" ]; then + __secure=${KERBEROS_ENABLED:-$(cdap_get_conf "kerberos.auth.enabled" "${CDAP_CONF}"/cdap-site.xml false)} + if [[ "${__secure}" == "true" ]]; then + cdap_kinit || return 1 + fi + + if [[ $(which hive 2>/dev/null) ]]; then + ERR_FILE=$(mktemp) + HIVE_VAR_OUT=$(hive -e 'set -v' 2>${ERR_FILE}) + __ret=$? + HIVE_ERR_MSG=$(< ${ERR_FILE}) + rm ${ERR_FILE} + if [ ${__ret} -ne 0 ]; then + echo "ERROR - While determining Hive classpath, failed to get Hive settings using: hive -e 'set -v'" + echo "If you do not want run CDAP with Hive functionality, set the 'explore.enabled' property in cdap-site.xml to 'false'" + echo "Otherwise, check that the Hive client is installed, and that Hive and HDFS are running." + echo "stderr:" + echo "${HIVE_ERR_MSG}" + return 1 + fi + HIVE_VARS=$(echo ${HIVE_VAR_OUT} | tr ' ' '\n') + # Quotes preserve whitespace + HIVE_HOME=${HIVE_HOME:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_HOME=' | cut -d= -f2)} + HIVE_CONF_DIR=${HIVE_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_CONF_DIR=' | cut -d= -f2)} + HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HADOOP_CONF_DIR=' | cut -d= -f2)} + fi + fi + + # If Hive classpath is successfully determined, derive explore + # classpath from it and export it to use it in the launch command + if [ -n "${HIVE_HOME}" -a -n "${HIVE_CONF_DIR}" -a -n "${HADOOP_CONF_DIR}" ]; then + EXPLORE_CONF_FILES=$(ls -1dF ${HIVE_CONF_DIR}/* ${HADOOP_CONF_DIR}/* | sed -e '/\/$/d' | tr '\n' ':') + EXPLORE_CLASSPATH=$(ls -1 ${HIVE_HOME}/lib/hive-exec-* ${HIVE_HOME}/lib/*.jar | tr '\n' ':') + export EXPLORE_CONF_FILES EXPLORE_CLASSPATH + fi + fi +} + +# Check that directory /var/tmp/cdap exists in the master node, or create it +cdap_check_or_create_master_local_dir() { + mkdir -p "${LOCAL_DIR}" +} + +# check and set classpath if in development enviroment +cdap_check_and_set_classpath_for_dev_environment () { + APP_HOME=${1} + + # Detect if we are in development. + IN_DEV_ENVIRONMENT=${IN_DEV_ENVIRONMENT:-false} + + # for developers only, add flow and flow related stuff to class path. + if [[ "${IN_DEV_ENVIRONMENT}" == "true" ]]; then + echo "Constructing classpath for development environment ..." + [[ -f "${APP_HOME}"/build/generated-classpath ]] && CLASSPATH+=":$(<${APP_HOME}/build/generated-classpath)" + [[ -d "${APP_HOME}"/build/classes ]] && CLASSPATH+=":${APP_HOME}/build/classes/main:${APP_HOME}/conf/*" + [[ -d "${APP_HOME}"/../data-fabric/build/classes ]] && CLASSPATH+=":${APP_HOME}/../data-fabric/build/classes/main" + [[ -d "${APP_HOME}"/../common/build/classes ]] && CLASSPATH+=":${APP_HOME}/../common/build/classes/main" + [[ -d "${APP_HOME}"/../gateway/build/classes ]] && CLASSPATH+=":${APP_HOME}/../gateway/build/classes/main" + export CLASSPATH + fi +} + +HOSTNAME=$(hostname -f) +export LOG_PREFIX=${APP}-${IDENT_STRING}-${HOSTNAME} +export LOGFILE=${LOG_PREFIX}.log +loglog="${LOG_DIR}/${LOGFILE}" + +pid=${PID_DIR}/${APP}-${IDENT_STRING}.pid +loggc="${LOG_DIR}/${LOG_PREFIX}.gc" + +export NICENESS=${NICENESS:-0} diff --git a/cdap3vm/config/cdap-config-template/logback-container.xml b/cdap3vm/config/cdap-config-template/logback-container.xml new file mode 100644 index 0000000..f1957d8 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback-container.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Copyright © 2015 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> + +<!-- + Logback used by containers that run in YARN eg. Flows, MR, Explore +--> +<configuration> + + <!-- + Disabling some chatty loggers. + --> + <logger name="org.apache.commons.beanutils" level="ERROR"/> + <logger name="org.apache.zookeeper.server" level="ERROR"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="com.ning" level="WARN"/> + <logger name="org.apache.spark" level="WARN"/> + <logger name="org.spark-project" level="WARN"/> + <logger name="org.apache.hadoop" level="WARN"/> + <logger name="org.apache.hive" level="WARN"/> + <logger name="org.quartz.core" level="WARN"/> + <logger name="org.eclipse.jetty" level="WARN"/> + <logger name="io.netty.util.internal" level="WARN"/> + + <logger name="org.apache.twill" level="WARN"/> + <logger name="co.cask.cdap" level="INFO"/> + + <!-- Redirected stdout and stderr of Hive operations --> + <logger name="Explore.stdout" level="INFO"/> + <logger name="Explore.stderr" level="INFO"/> + + + <!-- quick workaround suggested by CASK for ticket #666 [DE257314] --> + <logger name="org.apache.hadoop.io.retry.RetryInvocationHandler" level="ERROR"/> + + <appender name="Rolling" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- LOG_DIRS is the environment variable set by YARN for container logs --> + <file>${LOG_DIRS}/program.log</file> + <encoder> + <pattern>%d{ISO8601} - %-5p [%t:%logger{1}@%L] - %m%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- Daily rollover at midnight--> + <fileNamePattern>${LOG_DIRS}/program.%d.log</fileNamePattern> + + <!-- Keep 2 weeks of history --> + <maxHistory>14</maxHistory> + </rollingPolicy> + </appender> + + <root level="INFO"> + <appender-ref ref="Rolling"/> + </root> + +</configuration> + diff --git a/cdap3vm/config/cdap-config-template/logback.xml b/cdap3vm/config/cdap-config-template/logback.xml new file mode 100644 index 0000000..e036683 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback.xml @@ -0,0 +1,81 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Copyright © 2014 Cask Data, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); you may not + use this file except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations under + the License. + --> + +<configuration> + + <!-- + Disabling some chatty loggers. + --> + <logger name="org.apache.commons.beanutils" level="ERROR"/> + <logger name="org.apache.zookeeper.server" level="ERROR"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="com.ning" level="WARN"/> + <logger name="org.apache.spark" level="WARN"/> + <logger name="org.apache.hadoop" level="WARN"/> + <logger name="org.apache.hive" level="WARN"/> + <logger name="org.quartz.core" level="WARN"/> + <logger name="org.eclipse.jetty" level="WARN"/> + <logger name="org.apache.twill" level="WARN"/> + <logger name="co.cask.cdap" level="INFO"/> + + <appender name="Console" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern> + </encoder> + </appender> + + <root level="INFO"> + <appender-ref ref="Console"/> + </root> + + <!-- + Uncomment the following section to enable access logging by the Authentication Server and Router processes. + Log output will be in the standard Apache HTTPd access log format (see + http://httpd.apache.org/docs/2.2/logs.html#accesslog). + --> + <!-- + <appender name="AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>access.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>access.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <logger name="http-access" level="TRACE" additivity="false"> + <appender-ref ref="AUDIT" /> + </logger> + <appender name="EXTERNAL_AUTH_AUDIT" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>external_auth_access.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>external_auth_access.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <logger name="external-auth-access" level="TRACE" additivity="false"> + <appender-ref ref="EXTERNAL_AUTH_AUDIT" /> + </logger> + --> + +</configuration> + diff --git a/cdap3vm/config/hadoop-cluster-conf-file.sh b/cdap3vm/config/hadoop-cluster-conf-file.sh new file mode 100644 index 0000000..a5fcf3b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-conf-file.sh @@ -0,0 +1,131 @@ +#!/bin/sh + +# +# Based on HortonWorks Directories Script +# + +# __HDP_VERSION__=2.3.2.0-2950 +__HDP_VERSION__=2.4.3.0-227 +# Space separated list of directories where NameNode will store file system image. +# For example, /grid/hadoop/hdfs/nn /grid1/hadoop/hdfs/nn +__DFS_NAME_DIR__=/opt/data/hadoop/hdfs/namenode + +# Space separated list of directories where DataNodes will store the blocks. +# For example, /grid/hadoop/hdfs/dn /grid1/hadoop/hdfs/dn /grid2/hadoop/hdfs/dn +__DFS_DATA_DIR__=/opt/data/hadoop/hdfs/data + +# Space separated list of directories where SecondaryNameNode will store checkpoint image. +# For example, /grid/hadoop/hdfs/snn /grid1/hadoop/hdfs/snn /grid2/hadoop/hdfs/snn +__FS_CHECKPOINT_DIR__=/opt/data/hadoop/hdfs/snamenode + +# Directory to store the HDFS logs. +__HDFS_LOG_DIR__=/opt/data/log/hadoop/hdfs + +# Directory to store the HDFS process ID. +__HDFS_PID_DIR__=/var/run/hadoop/hdfs + +# Directory to store the Hadoop configuration files. +__HADOOP_CONF_DIR__=/etc/hadoop/conf + +# Main Hadoop LOG dir +__HADOOP_LOG_MAIN_DIR__=/opt/data/log + +# +# Hadoop Service - YARN +# + +# Space separated list of directories where YARN will store temporary data. +# For example, /grid/hadoop/yarn/local /grid1/hadoop/yarn/local /grid2/hadoop/yarn/local +__YARN_LOCAL_DIR__=/opt/data/hadoop/yarn/local + +# Directory to store the YARN logs. +__YARN_LOG_DIR__=/opt/data/log/hadoop/yarn + +# Directory for nodemanager.recovery +__YARN_NODEMANAGER_RECOVERY_DIR__=/opt/data/log/hadoop-yarn/nodemanager/recovery-state + + +# Space separated list of directories where YARN will store container log data. +# For example, /grid/hadoop/yarn/logs /grid1/hadoop/yarn/logs /grid2/hadoop/yarn/logs +__YARN_LOCAL_LOG_DIR__=/opt/data/hadoop/yarn/log + +# Directory to store the YARN process ID. +__YARN_PID_DIR__=/var/run/hadoop/yarn + +# +# Hadoop Service - MAPREDUCE +# + +# Directory to store the MapReduce daemon logs. +__MAPRED_LOG_DIR__=/opt/data/log/hadoop/mapred + +# Directory to store the mapreduce jobhistory process ID. +__MAPRED_PID_DIR__=/var/run/hadoop/mapred + +# +# Hadoop Service - HBase +# + +# Directory to store the HBase configuration files. +__HBASE_CONF_DIR__=/etc/hbase/conf + +# Directory to store the HBase logs. +__HBASE_LOG_DIR__=/opt/data/log/hbase + +# Directory to store the HBase logs. +__HBASE_PID_DIR__=/var/run/hbase + +# +# Hadoop Service - ZooKeeper +# + +# Directory where ZooKeeper will store data. For example, /grid1/hadoop/zookeeper/data +__ZOOKEEPER_DATA_DIR__=/opt/data/hadoop/zookeeper/data + +# Directory to store the ZooKeeper configuration files. +__ZOOKEEPER_CONF_DIR__=/etc/zookeeper/conf + +# Directory to store the ZooKeeper logs. +__ZOOKEEPER_LOG_DIR__=/opt/data/log/zookeeper +__ZKFC_LOG_DIR__=/opt/data/log/hdfs + +# Directory to store the ZooKeeper process ID. +__ZOOKEEPER_PID_DIR__=/var/run/zookeeper + + +# +# User Accounts +# + +__SUPER_USER__=root +__HADOOP_USER__=hadoop +__HDFS_USER__=hdfs +__YARN_USER__=yarn +__HBASE_USER__=hbase +__CDAP_USER__=cdap +__ZOOKEEPER_USER__=zookeeper +__MAPRED_USER__=mapred +__HADOOP_GROUP__=hadoop + +# +# Other service folder locations +# +__SERVICE_CONFIG_FOLDER__=/etc/hadoop/service_scripts +__HDP_CURRENT_FOLDER__=/usr/hdp/current +__HDP_HADOOP_BIN__=/usr/hdp/current/hadoop-client/bin +__HDP_HADOOP_SBIN__=/usr/hdp/current/hadoop-client/sbin +__RPM_FOLDER_LOCATION__=pkgs + +# __RPM_FOLDER_LOCATION__ is relative to main package directory +__HDP_RPM_ROOT__=UNUSED +__CDAP_RPM_ROOT__=UNUSED +__CENTOS7_RPM_ROOT__=UNUSED +# +# Main System +# +__JAVA_HOME__=/opt/app/java/jdk/jdk170 +__CDAP_CONF_DIR__=/etc/cdap/conf +__CDAP_INST_FOLDER__=/opt/cdap +__NODEJS_BIN__=/opt/app/nodejs/bin + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml new file mode 100644 index 0000000..0ab575e --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml @@ -0,0 +1,79 @@ +<!--Thu Jun 18 04:06:19 2015--> + <configuration> + + <property> + <name>yarn.scheduler.capacity.default.minimum-user-limit-percent</name> + <value>100</value> + </property> + + <property> + <name>yarn.scheduler.capacity.maximum-am-resource-percent</name> + <value>0.2</value> + </property> + + <property> + <name>yarn.scheduler.capacity.maximum-applications</name> + <value>10000</value> + </property> + + <property> + <name>yarn.scheduler.capacity.node-locality-delay</name> + <value>40</value> + </property> + + <property> + <name>yarn.scheduler.capacity.resource-calculator</name> + <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.accessible-node-labels</name> + <value>*</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.acl_administer_queue</name> + <value>*</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.capacity</name> + <value>100</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name> + <value>*</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name> + <value>*</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.capacity</name> + <value>100</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.maximum-capacity</name> + <value>100</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.state</name> + <value>RUNNING</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.default.user-limit-factor</name> + <value>1</value> + </property> + + <property> + <name>yarn.scheduler.capacity.root.queues</name> + <value>default</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg new file mode 100644 index 0000000..d009d0d --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg @@ -0,0 +1,23 @@ + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +yarn.nodemanager.local-dirs=/grid/0/hadoop/yarn/local +yarn.nodemanager.log-dirs=/grid/0/hadoop/yarn/log +yarn.nodemanager.linux-container-executor.group=hadoop +banned.users=hdfs,yarn,mapred,bin +min.user.id=500 diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml new file mode 100644 index 0000000..83bfc2c --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml @@ -0,0 +1,147 @@ +<!--Thu Jun 18 04:06:19 2015--> + <configuration> + + <property> + <name>fs.defaultFS</name> + <value>hdfs://__HDP_NAMENODE__:8020</value> + <final>true</final> + </property> + + <property> + <name>fs.trash.interval</name> + <value>360</value> + </property> + + <property> + <name>ha.failover-controller.active-standby-elector.zk.op.retries</name> + <value>120</value> + </property> + + <property> + <name>hadoop.http.authentication.simple.anonymous.allowed</name> + <value>true</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.hosts</name> + <value>__HDP_WEBHCAT_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.hosts</name> + <value>__HDP_HIVE_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.hosts</name> + <value>__HDP_OOZIE_SERVER__</value> + </property> + + <property> + <name>hadoop.security.auth_to_local</name> + <value>DEFAULT</value> + </property> + + <property> + <name>hadoop.security.authentication</name> + <value>simple</value> + </property> + + <property> + <name>hadoop.security.authorization</name> + <value>false</value> + </property> + + <property> + <name>hadoop.security.key.provider.path</name> + <value></value> + </property> + + <property> + <name>io.compression.codecs</name> + <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value> + </property> + + <property> + <name>io.file.buffer.size</name> + <value>131072</value> + </property> + + <property> + <name>io.serializations</name> + <value>org.apache.hadoop.io.serializer.WritableSerialization</value> + </property> + + <property> + <name>ipc.client.connect.max.retries</name> + <value>50</value> + </property> + + <property> + <name>ipc.client.connection.maxidletime</name> + <value>30000</value> + </property> + + <property> + <name>ipc.client.idlethreshold</name> + <value>8000</value> + </property> + + <property> + <name>ipc.server.tcpnodelay</name> + <value>true</value> + </property> + + <property> + <name>mapreduce.jobtracker.webinterface.trusted</name> + <value>false</value> + </property> + +<!-- + <property> + <name>net.topology.script.file.name</name> + <value>/etc/hadoop/conf/topology_script.py</value> + </property> +--> + + <property> + <name>proxyuser_group</name> + <value>users</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh new file mode 100644 index 0000000..f4a200c --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh @@ -0,0 +1,125 @@ + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +export JAVA_HOME=__JAVA_HOME__ +export HADOOP_HOME_WARN_SUPPRESS=1 + +export HADOOP_LOG_MAIN=__HADOOP_LOG_MAIN_DIR__ +# Hadoop home directory +export HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} + +# Hadoop Configuration Directory + + +# Path to jsvc required by secure HDP 2.0 datanode +export JSVC_HOME=/usr/lib/bigtop-utils + + +# The maximum amount of heap to use, in MB. Default is 1000. +export HADOOP_HEAPSIZE="1024" + +export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m" + +# Extra Java runtime options. Empty by default. +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" + +# Command specific options appended to HADOOP_OPTS when specified +HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" + +HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" + + +SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=128m -XX:MaxNewSize=128m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" +export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" +export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}" + +export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" + + +HADOOP_NFS3_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}" +HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}" + + +# On secure datanodes, user to run the datanode as after dropping privileges +export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-""} + +# Extra ssh options. Empty by default. +export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" + +# Where log files are stored. $HADOOP_HOME/logs by default. +export HADOOP_LOG_DIR=${HADOOP_LOG_MAIN}/$USER + +# History server logs +export HADOOP_MAPRED_LOG_DIR=${HADOOP_LOG_MAIN}-mapreduce/$USER + +# Where log files are stored in the secure data environment. +export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_MAIN}/$HADOOP_SECURE_DN_USER + +# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. +# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + +# host:path where hadoop code should be rsync'd from. Unset by default. +# export HADOOP_MASTER=master:/home/$USER/src/hadoop + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HADOOP_SLAVE_SLEEP=0.1 + +# The directory where pid files are stored. /tmp by default. +export HADOOP_PID_DIR=/var/run/hadoop/$USER +export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER + +# History server pid +export HADOOP_MAPRED_PID_DIR=/var/run/hadoop-mapreduce/$USER + +YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. + +# export HADOOP_NICENESS=10 + +# Use libraries from standard classpath +JAVA_JDBC_LIBS="" +#Add libraries required by mysql connector +for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +# Add libraries required by oracle connector +for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +# Add libraries required by nodemanager +MAPREDUCE_LIBS=/usr/hdp/current/hadoop-mapreduce-client/* +export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} + +# added to the HADOOP_CLASSPATH +if [ -d "/usr/hdp/current/tez-client" ]; then + if [ -d "/usr/hdp/current/tez-client/conf/" ]; then + # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP. + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/ + fi +fi + + +# Setting path to hdfs command line +export HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec + +# Mostly required for hadoop 2.0 +export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} + +export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml new file mode 100644 index 0000000..f6e8acf --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml @@ -0,0 +1,59 @@ +<!--Thu Jun 18 04:04:31 2015--> + <configuration> + + <property> + <name>security.admin.operations.protocol.acl</name> + <value>hadoop</value> + </property> + + <property> + <name>security.client.datanode.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.datanode.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.inter.datanode.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.inter.tracker.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.job.client.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.job.task.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.namenode.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.refresh.policy.protocol.acl</name> + <value>hadoop</value> + </property> + + <property> + <name>security.refresh.usertogroups.mappings.protocol.acl</name> + <value>hadoop</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml new file mode 100644 index 0000000..a9c9c0f --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml @@ -0,0 +1,281 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>dfs.block.access.token.enable</name> + <value>true</value> + </property> + + <property> + <name>dfs.blockreport.initialDelay</name> + <value>120</value> + </property> + + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit</name> + <value>true</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit.streams.cache.size</name> + <value>4096</value> + </property> + + <property> + <name>dfs.client.retry.policy.enabled</name> + <value>false</value> + </property> + + <property> + <name>dfs.cluster.administrators</name> + <value> hdfs</value> + </property> + + <property> + <name>dfs.datanode.address</name> + <value>0.0.0.0:50010</value> + </property> + + <property> + <name>dfs.datanode.balance.bandwidthPerSec</name> + <value>6250000</value> + </property> + + <property> + <name>dfs.datanode.data.dir</name> + <value>__DFS_DATA_DIR__</value> + </property> + + <property> + <name>dfs.datanode.data.dir.perm</name> + <value>750</value> + </property> + + <property> + <name>dfs.datanode.du.reserved</name> + <value>1073741824</value> + </property> + + <property> + <name>dfs.datanode.failed.volumes.tolerated</name> + <value>0</value> + </property> + + <property> + <name>dfs.datanode.http.address</name> + <value>0.0.0.0:50075</value> + </property> + + <property> + <name>dfs.datanode.https.address</name> + <value>0.0.0.0:50475</value> + </property> + + <property> + <name>dfs.datanode.ipc.address</name> + <value>0.0.0.0:8010</value> + </property> + + <property> + <name>dfs.datanode.max.transfer.threads</name> + <value>16384</value> + </property> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + </property> + + <property> + <name>dfs.encryption.key.provider.uri</name> + <value></value> + </property> + + <property> + <name>dfs.heartbeat.interval</name> + <value>3</value> + </property> + + <property> + <name>dfs.hosts.exclude</name> + <value>/etc/hadoop/conf/dfs.exclude</value> + </property> + + <property> + <name>dfs.http.policy</name> + <value>HTTP_ONLY</value> + </property> + + <property> + <name>dfs.https.port</name> + <value>50470</value> + </property> + + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/hadoop/hdfs/journalnode</value> + </property> + + <property> + <name>dfs.journalnode.http-address</name> + <value>0.0.0.0:8480</value> + </property> + + <property> + <name>dfs.journalnode.https-address</name> + <value>0.0.0.0:8481</value> + </property> + + <property> + <name>dfs.namenode.accesstime.precision</name> + <value>0</value> + </property> + + <property> + <name>dfs.namenode.audit.log.async</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.read.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.write.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.dir</name> + <value>__FS_CHECKPOINT_DIR__</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.edits.dir</name> + <value>${dfs.namenode.checkpoint.dir}</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.period</name> + <value>21600</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.txns</name> + <value>1000000</value> + </property> + + <property> + <name>dfs.namenode.fslock.fair</name> + <value>false</value> + </property> + + <property> + <name>dfs.namenode.handler.count</name> + <value>200</value> + </property> + + <property> + <name>dfs.namenode.http-address</name> + <value>__HDP_NAMENODE__:50070</value> + <final>true</final> + </property> + + <property> + <name>dfs.namenode.https-address</name> + <value>__HDP_NAMENODE__:50470</value> + </property> + + <property> + <name>dfs.namenode.name.dir</name> + <value>__DFS_NAME_DIR__</value> + </property> + + <property> + <name>dfs.namenode.name.dir.restore</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.rpc-address</name> + <value>__HDP_NAMENODE__:8020</value> + </property> + + <property> + <name>dfs.namenode.safemode.threshold-pct</name> + <value>1</value> + </property> + + <property> + <name>dfs.namenode.secondary.http-address</name> + <value>__HDP_SECONDARY_NAMENODE__:50090</value> + </property> + + <property> + <name>dfs.namenode.stale.datanode.interval</name> + <value>30000</value> + </property> + + <property> + <name>dfs.namenode.startup.delay.block.deletion.sec</name> + <value>3600</value> + </property> + + <property> + <name>dfs.namenode.write.stale.datanode.ratio</name> + <value>1.0f</value> + </property> + + <property> + <name>dfs.permissions.enabled</name> + <value>true</value> + </property> + + <property> + <name>dfs.permissions.superusergroup</name> + <value>hdfs</value> + </property> + + <property> + <name>dfs.replication</name> + <value>3</value> + </property> + + <property> + <name>dfs.replication.max</name> + <value>50</value> + </property> + + <property> + <name>dfs.support.append</name> + <value>true</value> + <final>true</final> + </property> + + <property> + <name>dfs.webhdfs.enabled</name> + <value>false</value> + </property> + + <property> + <name>fs.permissions.umask-mode</name> + <value>022</value> + </property> + + <property> + <name>nfs.exports.allowed.hosts</name> + <value>* rw</value> + </property> + + <property> + <name>nfs.file.dump.dir</name> + <value>/tmp/.hdfs-nfs</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check new file mode 100644 index 0000000..909d249 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check @@ -0,0 +1,64 @@ + +#!/bin/bash +# +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +err=0; + +function check_disks { + + for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do + fsdev="" + fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`; + if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then + msg_="$msg_ $m(u)" + else + msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`" + fi + done + + if [ -z "$msg_" ] ; then + echo "disks ok" ; exit 0 + else + echo "$msg_" ; exit 2 + fi + +} + +# Run all checks +for check in disks ; do + msg=`check_${check}` ; + if [ $? -eq 0 ] ; then + ok_msg="$ok_msg$msg," + else + err_msg="$err_msg$msg," + fi +done + +if [ ! -z "$err_msg" ] ; then + echo -n "ERROR $err_msg " +fi +if [ ! -z "$ok_msg" ] ; then + echo -n "OK: $ok_msg" +fi + +echo + +# Success! +exit 0 diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml new file mode 100644 index 0000000..cba69f4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml @@ -0,0 +1,135 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <!-- This file is hot-reloaded when it changes --> + + <!-- KMS ACLs --> + + <property> + <name>hadoop.kms.acl.CREATE</name> + <value>*</value> + <description> + ACL for create-key operations. + If the user is not in the GET ACL, the key material is not returned + as part of the response. + </description> + </property> + + <property> + <name>hadoop.kms.acl.DELETE</name> + <value>*</value> + <description> + ACL for delete-key operations. + </description> + </property> + + <property> + <name>hadoop.kms.acl.ROLLOVER</name> + <value>*</value> + <description> + ACL for rollover-key operations. + If the user is not in the GET ACL, the key material is not returned + as part of the response. + </description> + </property> + + <property> + <name>hadoop.kms.acl.GET</name> + <value>*</value> + <description> + ACL for get-key-version and get-current-key operations. + </description> + </property> + + <property> + <name>hadoop.kms.acl.GET_KEYS</name> + <value>*</value> + <description> + ACL for get-keys operations. + </description> + </property> + + <property> + <name>hadoop.kms.acl.GET_METADATA</name> + <value>*</value> + <description> + ACL for get-key-metadata and get-keys-metadata operations. + </description> + </property> + + <property> + <name>hadoop.kms.acl.SET_KEY_MATERIAL</name> + <value>*</value> + <description> + Complementary ACL for CREATE and ROLLOVER operations to allow the client + to provide the key material when creating or rolling a key. + </description> + </property> + + <property> + <name>hadoop.kms.acl.GENERATE_EEK</name> + <value>*</value> + <description> + ACL for generateEncryptedKey CryptoExtension operations. + </description> + </property> + + <property> + <name>hadoop.kms.acl.DECRYPT_EEK</name> + <value>*</value> + <description> + ACL for decryptEncryptedKey CryptoExtension operations. + </description> + </property> + + <property> + <name>default.key.acl.MANAGEMENT</name> + <value>*</value> + <description> + default ACL for MANAGEMENT operations for all key acls that are not + explicitly defined. + </description> + </property> + + <property> + <name>default.key.acl.GENERATE_EEK</name> + <value>*</value> + <description> + default ACL for GENERATE_EEK operations for all key acls that are not + explicitly defined. + </description> + </property> + + <property> + <name>default.key.acl.DECRYPT_EEK</name> + <value>*</value> + <description> + default ACL for DECRYPT_EEK operations for all key acls that are not + explicitly defined. + </description> + </property> + + <property> + <name>default.key.acl.READ</name> + <value>*</value> + <description> + default ACL for READ operations for all key acls that are not + explicitly defined. + </description> + </property> + + +</configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh new file mode 100644 index 0000000..44dfe6a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# Set kms specific environment variables here. + +# Settings for the Embedded Tomcat that runs KMS +# Java System properties for KMS should be specified in this variable +# +# export CATALINA_OPTS= + +# KMS logs directory +# +# export KMS_LOG=${KMS_HOME}/logs + +# KMS temporary directory +# +# export KMS_TEMP=${KMS_HOME}/temp + +# The HTTP port used by KMS +# +# export KMS_HTTP_PORT=16000 + +# The Admin port used by KMS +# +# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1` + +# The maximum number of Tomcat handler threads +# +# export KMS_MAX_THREADS=1000 + +# The location of the SSL keystore if using SSL +# +# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore + +# The password of the SSL keystore if using SSL +# +# export KMS_SSL_KEYSTORE_PASS=password + +# The full path to any native libraries that need to be loaded +# (For eg. location of natively compiled tomcat Apache portable +# runtime (APR) libraries +# +# export JAVA_LIBRARY_PATH=${HOME}/lib/native diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml new file mode 100644 index 0000000..a810ca4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml @@ -0,0 +1,173 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration> + + <!-- KMS Backend KeyProvider --> + + <property> + <name>hadoop.kms.key.provider.uri</name> + <value>jceks://file@/${user.home}/kms.keystore</value> + <description> + URI of the backing KeyProvider for the KMS. + </description> + </property> + + <property> + <name>hadoop.security.keystore.JavaKeyStoreProvider.password</name> + <value>none</value> + <description> + If using the JavaKeyStoreProvider, the password for the keystore file. + </description> + </property> + + <!-- KMS Cache --> + + <property> + <name>hadoop.kms.cache.enable</name> + <value>true</value> + <description> + Whether the KMS will act as a cache for the backing KeyProvider. + When the cache is enabled, operations like getKeyVersion, getMetadata, + and getCurrentKey will sometimes return cached data without consulting + the backing KeyProvider. Cached values are flushed when keys are deleted + or modified. + </description> + </property> + + <property> + <name>hadoop.kms.cache.timeout.ms</name> + <value>600000</value> + <description> + Expiry time for the KMS key version and key metadata cache, in + milliseconds. This affects getKeyVersion and getMetadata. + </description> + </property> + + <property> + <name>hadoop.kms.current.key.cache.timeout.ms</name> + <value>30000</value> + <description> + Expiry time for the KMS current key cache, in milliseconds. This + affects getCurrentKey operations. + </description> + </property> + + <!-- KMS Audit --> + + <property> + <name>hadoop.kms.audit.aggregation.window.ms</name> + <value>10000</value> + <description> + Duplicate audit log events within the aggregation window (specified in + ms) are quashed to reduce log traffic. A single message for aggregated + events is printed at the end of the window, along with a count of the + number of aggregated events. + </description> + </property> + + <!-- KMS Security --> + + <property> + <name>hadoop.kms.authentication.type</name> + <value>simple</value> + <description> + Authentication type for the KMS. Can be either "simple" + or "kerberos". + </description> + </property> + + <property> + <name>hadoop.kms.authentication.kerberos.keytab</name> + <value>${user.home}/kms.keytab</value> + <description> + Path to the keytab with credentials for the configured Kerberos principal. + </description> + </property> + + <property> + <name>hadoop.kms.authentication.kerberos.principal</name> + <value>HTTP/localhost</value> + <description> + The Kerberos principal to use for the HTTP endpoint. + The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification. + </description> + </property> + + <property> + <name>hadoop.kms.authentication.kerberos.name.rules</name> + <value>DEFAULT</value> + <description> + Rules used to resolve Kerberos principal names. + </description> + </property> + + <!-- Authentication cookie signature source --> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider</name> + <value>random</value> + <description> + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. + </description> + </property> + + <!-- Configuration for 'zookeeper' authentication cookie signature source --> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name> + <value>/hadoop-kms/hadoop-auth-signature-secret</value> + <description> + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. + </description> + </property> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name> + <value>#HOSTNAME#:#PORT#,...</value> + <description> + The Zookeeper connection string, a list of hostnames and port comma + separated. + </description> + </property> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name> + <value>kerberos</value> + <description> + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). + </description> + </property> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name> + <value>/etc/hadoop/conf/kms.keytab</value> + <description> + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. + </description> + </property> + + <property> + <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name> + <value>kms/#HOSTNAME#</value> + <description> + The Kerberos service principal used to connect to Zookeeper. + </description> + </property> + +</configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties new file mode 100644 index 0000000..9ad03cd --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties @@ -0,0 +1,225 @@ + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# Define some default values that can be overridden by system properties +# To change daemon root logger use hadoop_root_logger in hadoop-env +hadoop.root.logger=INFO,console +hadoop.log.dir=. +hadoop.log.file=hadoop.log + + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hadoop.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# +# TaskLog Appender +# + +#Default values +hadoop.tasklog.taskid=null +hadoop.tasklog.iscleanup=false +hadoop.tasklog.noKeepSplits=4 +hadoop.tasklog.totalLogFileSize=100 +hadoop.tasklog.purgeLogSplits=true +hadoop.tasklog.logsRetainHours=12 + +log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender +log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} +log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} +log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +#Security audit appender +# +hadoop.security.logger=INFO,console +hadoop.security.log.maxfilesize=256MB +hadoop.security.log.maxbackupindex=20 +log4j.category.SecurityLogger=${hadoop.security.logger} +hadoop.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd + +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} + +# +# hdfs audit logging +# +hdfs.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} +log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false +log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log +log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd + +# +# mapred audit logging +# +mapred.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} +log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false +log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log +log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd + +# +# Rolling File Appender +# + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Logfile size and and 30-day backups +log4j.appender.RFA.MaxFileSize=256MB +log4j.appender.RFA.MaxBackupIndex=10 + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# Custom Logging levels + +hadoop.metrics.log.level=INFO +#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG +#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} + +# Jets3t library +log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR + +# +# Null Appender +# Trap security logger on the hadoop client side +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter + +# Removes "deprecated" messages +log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN + +# +# HDFS block state change log from block manager +# +# Uncomment the following to suppress normal block state change +# messages from BlockManager in NameNode. +#log4j.logger.BlockStateChange=WARN +#Relative to Yarn Log Dir Prefix +yarn.log.dir=. +# +# Job Summary Appender +# +# Use following logger to send summary to separate file defined by +# hadoop.mapreduce.jobsummary.log.file rolled daily: +# hadoop.mapreduce.jobsummary.logger=INFO,JSA +# +hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log +log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender +# Set the ResourceManager summary log filename +yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log +# Set the ResourceManager summary log level and appender +yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} +#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY + +# To enable AppSummaryLogging for the RM, +# set yarn.server.resourcemanager.appsummary.logger to +# LEVEL,RMSUMMARY in hadoop-env.sh + +# Appender for ResourceManager Application Summary Log +# Requires the following properties to be set +# - hadoop.log.dir (Hadoop Log directory) +# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) +# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) +log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender +log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} +log4j.appender.RMSUMMARY.MaxFileSize=256MB +log4j.appender.RMSUMMARY.MaxBackupIndex=20 +log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout +log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n +log4j.appender.JSA.DatePattern=.yyyy-MM-dd +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false + +# Appender for viewing information for errors and warnings +yarn.ewma.cleanupInterval=300 +yarn.ewma.messageAgeLimitSeconds=86400 +yarn.ewma.maxUniqueMessages=250 +log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender +log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} +log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} +log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh new file mode 100644 index 0000000..582cdd4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh @@ -0,0 +1,14 @@ + +export JAVA_HOME=__JAVA_HOME__ + +export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=900 + +export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA + +#export HADOOP_JOB_HISTORYSERVER_OPTS= +#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. +#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. +#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. +#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default +#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. +export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml new file mode 100644 index 0000000..58b9ae1 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml @@ -0,0 +1,224 @@ +<!--Thu Jun 18 04:06:19 2015--> + <configuration> + + <property> + <name>mapreduce.admin.map.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.reduce.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.user.env</name> + <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + </property> + + <property> + <name>mapreduce.am.max-attempts</name> + <value>2</value> + </property> + + <property> + <name>mapreduce.application.classpath</name> + <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value> + </property> + + <property> + <name>mapreduce.application.framework.path</name> + <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value> + </property> + + <property> + <name>mapreduce.cluster.administrators</name> + <value> hadoop</value> + </property> + + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + + <property> + <name>mapreduce.job.counters.max</name> + <value>130</value> + </property> + + <property> + <name>mapreduce.job.emit-timeline-data</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.job.reduce.slowstart.completedmaps</name> + <value>0.05</value> + </property> + + <property> + <name>mapreduce.jobhistory.address</name> + <value>__HDP_HISTORYSERVER__:10020</value> + </property> + + <property> + <name>mapreduce.jobhistory.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>mapreduce.jobhistory.done-dir</name> + <value>/mr-history/done</value> + </property> + + <property> + <name>mapreduce.jobhistory.intermediate-done-dir</name> + <value>/mr-history/tmp</value> + </property> + + <property> + <name>mapreduce.jobhistory.webapp.address</name> + <value>__HDP_HISTORYSERVER__:19888</value> + </property> + + <property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx1024m</value> + </property> + + <property> + <name>mapreduce.map.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.map.memory.mb</name> + <value>2560</value> + </property> + + <property> + <name>mapreduce.map.output.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.map.sort.spill.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.map.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress.type</name> + <value>BLOCK</value> + </property> + + <property> + <name>mapreduce.reduce.input.buffer.percent</name> + <value>0.0</value> + </property> + + <property> + <name>mapreduce.reduce.java.opts</name> + <value>-Xmx4096m</value> + </property> + + <property> + <name>mapreduce.reduce.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.reduce.memory.mb</name> + <value>5120</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name> + <value>1</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name> + <value>1000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name> + <value>30000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.input.buffer.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.merge.percent</name> + <value>0.66</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.parallelcopies</name> + <value>30</value> + </property> + + <property> + <name>mapreduce.reduce.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.shuffle.port</name> + <value>13562</value> + </property> + + <property> + <name>mapreduce.task.io.sort.factor</name> + <value>100</value> + </property> + + <property> + <name>mapreduce.task.io.sort.mb</name> + <value>1433</value> + </property> + + <property> + <name>mapreduce.task.timeout</name> + <value>300000</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.admin-command-opts</name> + <value>-Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.command-opts</name> + <value>-Xmx2048m -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.resource.mb</name> + <value>2560</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.staging-dir</name> + <value>/user</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml new file mode 100644 index 0000000..df08d8e --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml @@ -0,0 +1,39 @@ +<!--Thu Jun 18 04:06:20 2015--> + <configuration> + + <property> + <name>ssl.client.keystore.location</name> + <value>/etc/security/clientKeys/keystore.jks</value> + </property> + + <property> + <name>ssl.client.keystore.password</name> + <value>bigdata</value> + </property> + + <property> + <name>ssl.client.keystore.type</name> + <value>jks</value> + </property> + + <property> + <name>ssl.client.truststore.location</name> + <value>/etc/security/clientKeys/all.jks</value> + </property> + + <property> + <name>ssl.client.truststore.password</name> + <value>bigdata</value> + </property> + + <property> + <name>ssl.client.truststore.reload.interval</name> + <value>10000</value> + </property> + + <property> + <name>ssl.client.truststore.type</name> + <value>jks</value> + </property> + + </configuration>
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml new file mode 100644 index 0000000..3ba6399 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml @@ -0,0 +1,44 @@ +<!--Thu Jun 18 04:06:20 2015--> + <configuration> + + <property> + <name>ssl.server.keystore.keypassword</name> + <value>bigdata</value> + </property> + + <property> + <name>ssl.server.keystore.location</name> + <value>/etc/security/serverKeys/keystore.jks</value> + </property> + + <property> + <name>ssl.server.keystore.password</name> + <value>bigdata</value> + </property> + + <property> + <name>ssl.server.keystore.type</name> + <value>jks</value> + </property> + + <property> + <name>ssl.server.truststore.location</name> + <value>/etc/security/serverKeys/all.jks</value> + </property> + + <property> + <name>ssl.server.truststore.password</name> + <value>bigdata</value> + </property> + + <property> + <name>ssl.server.truststore.reload.interval</name> + <value>10000</value> + </property> + + <property> + <name>ssl.server.truststore.type</name> + <value>jks</value> + </property> + + </configuration>
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg new file mode 100644 index 0000000..e0d5309 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg @@ -0,0 +1,21 @@ + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +mapred.local.dir=/tmp/hadoop-mapred/mapred/local +mapreduce.tasktracker.group=hadoop +hadoop.log.dir=__MAPRED_LOG_DIR__ diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh new file mode 100755 index 0000000..0f858c9 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh @@ -0,0 +1,119 @@ + + export HADOOP_YARN_HOME=/usr/hdp/current/hadoop-yarn-nodemanager + export YARN_LOG_DIR=__YARN_LOG_DIR__/$USER + export YARN_PID_DIR=__YARN_PID_DIR__/$USER + export HADOOP_LIBEXEC_DIR=__HDP_CURRENT_FOLDER__/hadoop-client/libexec + export JAVA_HOME=__JAVA_HOME__ + + # We need to add the EWMA appender for the yarn daemons only; + # however, YARN_ROOT_LOGGER is shared by the yarn client and the + # daemons. This is restrict the EWMA appender to daemons only. + INVOKER="${0##*/}" + if [ "$INVOKER" == "yarn-daemon.sh" ]; then + export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA} + fi + + # User for YARN daemons + export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + + # resolve links - $0 may be a softlink + export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" + + # some Java parameters + # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME + fi + + if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 + fi + + JAVA=$JAVA_HOME/bin/java + JAVA_HEAP_MAX=-Xmx1000m + + # For setting YARN specific HEAP sizes please use this + # Parameter and set appropriately + YARN_HEAPSIZE=1024 + + # check envvars which might override default args + if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" + fi + + # Resource Manager specific parameters + + # Specify the max Heapsize for the ResourceManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_RESOURCEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_RESOURCEMANAGER_HEAPSIZE=1024 + + # Specify the JVM options to be used when starting the ResourceManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_RESOURCEMANAGER_OPTS= + + # Node Manager specific parameters + + # Specify the max Heapsize for the NodeManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_NODEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_NODEMANAGER_HEAPSIZE=1024 + + # Specify the max Heapsize for the HistoryManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1024. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_HISTORYSERVER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_HISTORYSERVER_HEAPSIZE=1024 + + # Specify the JVM options to be used when starting the NodeManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_NODEMANAGER_OPTS= + + # so that filenames w/ spaces are handled correctly in loops below + IFS= + + + # default log directory and file + if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" + fi + if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' + fi + + # default policy file for service-level authorization + if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" + fi + + # restore ordinary behaviour + unset IFS + + + YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" + YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" + YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + fi + YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml new file mode 100644 index 0000000..bd96fb0 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml @@ -0,0 +1,527 @@ +<!--Thu Jun 18 04:14:19 2015--> + <configuration> + + <property> + <name>hadoop.registry.rm.enabled</name> + <value>true</value> + </property> + + <property> + <name>hadoop.registry.zk.quorum</name> + <value>__HDP_ZOOKEEPER_HOSTS_WITH_PORT__</value> + </property> + + <property> + <name>yarn.acl.enable</name> + <value>false</value> + </property> + + <property> + <name>yarn.admin.acl</name> + <value>yarn</value> + </property> + + <property> + <name>yarn.application.classpath</name> + <value>__HADOOP_CONF_DIR__,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*</value> + </property> + + <property> + <name>yarn.client.nodemanager-connect.max-wait-ms</name> + <value>60000</value> + </property> + + <property> + <name>yarn.client.nodemanager-connect.retry-interval-ms</name> + <value>10000</value> + </property> + + <property> + <name>yarn.http.policy</name> + <value>HTTP_ONLY</value> + </property> + + <property> + <name>yarn.log-aggregation-enable</name> + <value>true</value> + </property> + + <property> + <name>yarn.log-aggregation.retain-seconds</name> + <value>2592000</value> + </property> + + <property> + <name>yarn.log.server.url</name> + <value>http://__HDP_HISTORYSERVER__:19888/jobhistory/logs</value> + </property> + + <property> + <name>yarn.node-labels.enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.node-labels.fs-store.retry-policy-spec</name> + <value>2000, 500</value> + </property> + + <property> + <name>yarn.node-labels.fs-store.root-dir</name> + <value>/system/yarn/node-labels</value> + </property> + + <property> + <name>yarn.nodemanager.address</name> + <value>0.0.0.0:45454</value> + </property> + + <property> + <name>yarn.nodemanager.admin-env</name> + <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value> + </property> + + <property> + <name>yarn.nodemanager.aux-services</name> + <value>mapreduce_shuffle</value> + </property> + + <property> + <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name> + <value>org.apache.hadoop.mapred.ShuffleHandler</value> + </property> + + <property> + <name>yarn.nodemanager.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>yarn.nodemanager.container-executor.class</name> + <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value> + </property> + + <property> + <name>yarn.nodemanager.container-monitor.interval-ms</name> + <value>3000</value> + </property> + + <property> + <name>yarn.nodemanager.delete.debug-delay-sec</name> + <value>43200</value> + <!-- [DE239909] tune for DCAE CDAP --> + </property> + + <property> + <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name> + <value>90</value> + </property> + + <property> + <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name> + <value>1000</value> + </property> + + <property> + <name>yarn.nodemanager.disk-health-checker.min-healthy-disks</name> + <value>0.25</value> + </property> + + <property> + <name>yarn.nodemanager.health-checker.interval-ms</name> + <value>135000</value> + </property> + + <property> + <name>yarn.nodemanager.health-checker.script.timeout-ms</name> + <value>60000</value> + </property> + + <property> + <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name> + <value>hadoop-yarn</value> + </property> + + <property> + <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name> + <value>false</value> + </property> + + <property> + <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name> + <value>false</value> + </property> + + <property> + <name>yarn.nodemanager.linux-container-executor.group</name> + <value>hadoop</value> + </property> + + <property> + <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name> + <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value> + </property> + + <property> + <name>yarn.nodemanager.local-dirs</name> + <value>__YARN_LOCAL_DIR__</value> + </property> + + <property> + <name>yarn.nodemanager.log-aggregation.compression-type</name> + <value>gz</value> + </property> + + <property> + <name>yarn.nodemanager.log-aggregation.debug-enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name> + <value>30</value> + </property> + + <property> + <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name> + <value>-1</value> + </property> + + <property> + <name>yarn.nodemanager.log-dirs</name> + <value>__YARN_LOG_DIR__</value> + </property> + + <property> + <name>yarn.nodemanager.log.retain-second</name> + <value>604800</value> + </property> + + <property> + <name>yarn.nodemanager.recovery.dir</name> + <value>__YARN_NODEMANAGER_RECOVERY_DIR__</value> + </property> + + <property> + <name>yarn.nodemanager.recovery.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.nodemanager.remote-app-log-dir</name> + <value>/app-logs</value> + </property> + + <property> + <name>yarn.nodemanager.remote-app-log-dir-suffix</name> + <value>logs</value> + </property> + + <property> + <name>yarn.nodemanager.resource.cpu-vcores</name> + <value>6</value> + </property> + + <property> + <name>yarn.nodemanager.resource.memory-mb</name> + <value>15360</value> + </property> + + <property> + <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name> + <value>80</value> + </property> + + <property> + <name>yarn.nodemanager.vmem-check-enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.nodemanager.vmem-pmem-ratio</name> + <value>2.1</value> + </property> + + <property> + <name>yarn.resourcemanager.address</name> + <value>__HDP_RESOURCEMANAGER__:8050</value> + </property> + + <property> + <name>yarn.resourcemanager.admin.address</name> + <value>__HDP_RESOURCEMANAGER__:8141</value> + </property> + + <property> + <name>yarn.resourcemanager.am.max-attempts</name> + <value>2</value> + </property> + + <property> + <name>yarn.resourcemanager.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>yarn.resourcemanager.connect.max-wait.ms</name> + <value>900000</value> + </property> + + <property> + <name>yarn.resourcemanager.connect.retry-interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name> + <value>2000, 500</value> + </property> + + <property> + <name>yarn.resourcemanager.fs.state-store.uri</name> + <value> </value> + </property> + + <property> + <name>yarn.resourcemanager.ha.enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.resourcemanager.hostname</name> + <value>__HDP_RESOURCEMANAGER__</value> + </property> + + <property> + <name>yarn.resourcemanager.nodes.exclude-path</name> + <value>/etc/hadoop/conf/yarn.exclude</value> + </property> + + <property> + <name>yarn.resourcemanager.recovery.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.resourcemanager.resource-tracker.address</name> + <value>__HDP_RESOURCEMANAGER__:8025</value> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.address</name> + <value>__HDP_RESOURCEMANAGER__:8030</value> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.monitor.enable</name> + <value>false</value> + </property> + + <property> + <name>yarn.resourcemanager.state-store.max-completed-applications</name> + <value>${yarn.resourcemanager.max-completed-applications}</value> + </property> + + <property> + <name>yarn.resourcemanager.store.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> + </property> + + <property> + <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name> + <value>10</value> + </property> + + <property> + <name>yarn.resourcemanager.system-metrics-publisher.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.resourcemanager.webapp.address</name> + <value>__HDP_RESOURCEMANAGER__:8088</value> + </property> + + <property> + <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.resourcemanager.webapp.https.address</name> + <value>__HDP_RESOURCEMANAGER__:8090</value> + </property> + + <property> + <name>yarn.resourcemanager.work-preserving-recovery.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name> + <value>10000</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-acl</name> + <value>world:anyone:rwcda</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-address</name> + <value>__HDP_ZOOKEEPER_HOSTS_WITH_PORT__</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-num-retries</name> + <value>1000</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-retry-interval-ms</name> + <value>1000</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-state-store.parent-path</name> + <value>/rmstore</value> + </property> + + <property> + <name>yarn.resourcemanager.zk-timeout-ms</name> + <value>10000</value> + </property> + + <property> + <name>yarn.scheduler.maximum-allocation-mb</name> + <value>8192</value> + <!-- [DE239909] tune for DCAE CDAP --> + </property> + + <property> + <name>yarn.scheduler.maximum-allocation-vcores</name> + <value>2</value> + </property> + + <property> + <name>yarn.scheduler.minimum-allocation-mb</name> + <value>512</value> + <!-- [DE239909] tune for DCAE CDAP --> + </property> + + <property> + <name>yarn.scheduler.minimum-allocation-vcores</name> + <value>1</value> + </property> + + <property> + <name>yarn.timeline-service.address</name> + <value>__HDP_APP_TIMELINE_SERVER__:10200</value> + </property> + + <property> + <name>yarn.timeline-service.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>yarn.timeline-service.client.max-retries</name> + <value>30</value> + </property> + + <property> + <name>yarn.timeline-service.client.retry-interval-ms</name> + <value>1000</value> + </property> + + <property> + <name>yarn.timeline-service.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.generic-application-history.store-class</name> + <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value> + </property> + + <property> + <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.http-authentication.type</name> + <value>simple</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-state-store.path</name> + <value>/grid/0/hadoop/yarn/timeline</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-timeline-store.path</name> + <value>/grid/0/hadoop/yarn/timeline</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name> + <value>104857600</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name> + <value>10000</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name> + <value>10000</value> + </property> + + <property> + <name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name> + <value>300000</value> + </property> + + <property> + <name>yarn.timeline-service.recovery.enabled</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.state-store-class</name> + <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value> + </property> + + <property> + <name>yarn.timeline-service.store-class</name> + <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value> + </property> + + <property> + <name>yarn.timeline-service.ttl-enable</name> + <value>true</value> + </property> + + <property> + <name>yarn.timeline-service.ttl-ms</name> + <value>2678400000</value> + </property> + + <property> + <name>yarn.timeline-service.webapp.address</name> + <value>__HDP_APP_TIMELINE_SERVER__:8188</value> + </property> + + <property> + <name>yarn.timeline-service.webapp.https.address</name> + <value>__HDP_APP_TIMELINE_SERVER__:8190</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude @@ -0,0 +1 @@ + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml new file mode 100644 index 0000000..b021418 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml @@ -0,0 +1,147 @@ +<!--Thu Jun 18 04:06:19 2015--> + <configuration> + + <property> + <name>fs.defaultFS</name> + <value>hdfs://__HDP_NAMENODE__:8020</value> + <final>true</final> + </property> + + <property> + <name>fs.trash.interval</name> + <value>360</value> + </property> + + <property> + <name>ha.failover-controller.active-standby-elector.zk.op.retries</name> + <value>120</value> + </property> + + <property> + <name>hadoop.http.authentication.simple.anonymous.allowed</name> + <value>true</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.hosts</name> + <value>__HDP_WEBHCAT_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.hosts</name> + <value>__HDP_HIVE_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.hosts</name> + <value>__HDP_OOZIE_SERVER__</value> + </property> + + <property> + <name>hadoop.security.auth_to_local</name> + <value>DEFAULT</value> + </property> + + <property> + <name>hadoop.security.authentication</name> + <value>simple</value> + </property> + + <property> + <name>hadoop.security.authorization</name> + <value>false</value> + </property> + + <property> + <name>hadoop.security.key.provider.path</name> + <value></value> + </property> + + <property> + <name>io.compression.codecs</name> + <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value> + </property> + + <property> + <name>io.file.buffer.size</name> + <value>131072</value> + </property> + + <property> + <name>io.serializations</name> + <value>org.apache.hadoop.io.serializer.WritableSerialization</value> + </property> + + <property> + <name>ipc.client.connect.max.retries</name> + <value>50</value> + </property> + + <property> + <name>ipc.client.connection.maxidletime</name> + <value>30000</value> + </property> + + <property> + <name>ipc.client.idlethreshold</name> + <value>8000</value> + </property> + + <property> + <name>ipc.server.tcpnodelay</name> + <value>true</value> + </property> + + <property> + <name>mapreduce.jobtracker.webinterface.trusted</name> + <value>false</value> + </property> + +<!-- + <property> + <name>net.topology.script.file.name</name> + <value>/etc/hadoop/conf/topology_script.py</value> + </property> +--> + + <property> + <name>proxyuser_group</name> + <value>users</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..0911e46 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties @@ -0,0 +1,50 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + + +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.slave.host.name=rohit-companionfiles-1.c.pramod-thangali.internal +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd new file mode 100644 index 0000000..9fb87d7 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd @@ -0,0 +1,87 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Set environment variables here. + +@rem The java implementation to use. Java 1.7+ required. +@rem set JAVA_HOME=c:\apps\java + +@rem Extra Java CLASSPATH elements. Optional. +@rem set HBASE_CLASSPATH= + +@rem The maximum amount of heap to use. Default is left to JVM default. +@rem set HBASE_HEAPSIZE=1000 + +@rem Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +@rem offheap, set the value to "8G". +@rem set HBASE_OFFHEAPSIZE=1000 + +@rem For example, to allocate 8G of offheap, to 8G: +@rem etHBASE_OFFHEAPSIZE=8G + +@rem Extra Java runtime options. +@rem Below are what we set by default. May only work with SUN JVM. +@rem For more on why as well as other possible settings, +@rem see http://wiki.apache.org/hadoop/PerformanceTuning +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +@rem @rem See TestIPv6NIOServerSocketChannel. +set HBASE_OPTS="-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stack=true" + +@rem Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ +set HBASE_MASTER_OPTS=%HBASE_MASTER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" +set HBASE_REGIONSERVER_OPTS=%HBASE_REGIONSERVER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" + +@rem Uncomment below to enable java garbage collection logging for the server-side processes +@rem this enables basic gc logging for the server processes to the .out file +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" "-XX:+UseGCLogFileRotation" "-XX:NumberOfGCLogFiles=1" "-XX:GCLogFileSize=512M" %HBASE_GC_OPTS% + +@rem Uncomment below to enable java garbage collection logging for the client processes in the .out file. +@rem set CLIENT_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +@rem set HBASE_USE_GC_LOGFILE=true + +@rem Uncomment and adjust to enable JMX exporting +@rem See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +@rem More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +@rem +@rem set HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false" "-Dcom.sun.management.jmxremote.authenticate=false" +@rem set HBASE_MASTER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10101" +@rem set HBASE_REGIONSERVER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10102" +@rem set HBASE_THRIFT_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10103" +@rem set HBASE_ZOOKEEPER_OPTS=%HBASE_JMX_BASE% -Dcom.sun.management.jmxremote.port=10104" + +@rem File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +@rem set HBASE_REGIONSERVERS=%HBASE_HOME%\conf\regionservers + +@rem Where log files are stored. $HBASE_HOME/logs by default. +@rem set HBASE_LOG_DIR=%HBASE_HOME%\logs + +@rem A string representing this instance of hbase. $USER by default. +@rem set HBASE_IDENT_STRING=%USERNAME% + +@rem Seconds to sleep between slave commands. Unset by default. This +@rem can be useful in large clusters, where, e.g., slave rsyncs can +@rem otherwise arrive faster than the master can service them. +@rem set HBASE_SLAVE_SLEEP=0.1 + +@rem Tell HBase whether it should manage it's own instance of Zookeeper or not. +@rem set HBASE_MANAGES_ZK=true diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh new file mode 100644 index 0000000..ce60e00 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh @@ -0,0 +1,65 @@ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME=__JAVA_HOME__ + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-__HBASE_CONF_DIR__} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:__HBASE_LOG_DIR__/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR=__HBASE_LOG_DIR__ + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR=/var/run/hbase + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + + + + +export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile=__HBASE_LOG_DIR__/hs_err_pid%p.log" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx4096m $JDK_DEPENDED_OPTS" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn512m -XX:CMSInitiatingOccupancyFraction=70 -Xms1024m -Xmx8192m $JDK_DEPENDED_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml new file mode 100644 index 0000000..0392423 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml @@ -0,0 +1,19 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + </property> + + </configuration>
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml new file mode 100644 index 0000000..4c75e95 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml @@ -0,0 +1,234 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + </property> + + <property> + <name>hbase.bulkload.staging.dir</name> + <value>/apps/hbase/staging</value> + </property> + + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>1048576</value> + </property> + + <property> + <name>hbase.client.retries.number</name> + <value>35</value> + </property> + + <property> + <name>hbase.client.scanner.caching</name> + <value>100</value> + </property> + + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + + <property> + <name>hbase.coprocessor.master.classes</name> + <value></value> + </property> + + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value> + </property> + + <property> + <name>hbase.defaults.for.version.skip</name> + <value>true</value> + </property> + + <property> + <name>hbase.hregion.majorcompaction</name> + <value>604800000</value> + </property> + + <property> + <name>hbase.hregion.majorcompaction.jitter</name> + <value>0.50</value> + </property> + + <property> + <name>hbase.hregion.max.filesize</name> + <value>10737418240</value> + </property> + + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>4</value> + </property> + + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>134217728</value> + </property> + + <property> + <name>hbase.hregion.memstore.mslab.enabled</name> + <value>true</value> + </property> + + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>10</value> + </property> + + <property> + <name>hbase.hstore.compaction.max</name> + <value>10</value> + </property> + + <property> + <name>hbase.hstore.compactionThreshold</name> + <value>3</value> + </property> + + <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + </property> + + <property> + <name>hbase.master.info.bindAddress</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>hbase.master.info.port</name> + <value>16010</value> + </property> + + <property> + <name>hbase.master.port</name> + <value>16000</value> + </property> + + <property> + <name>hbase.region.server.rpc.scheduler.factory.class</name> + <value></value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.lowerLimit</name> + <value>0.38</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.size</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.upperLimit</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.handler.count</name> + <value>30</value> + </property> + + <property> + <name>hbase.regionserver.info.port</name> + <value>16030</value> + </property> + + <property> + <name>hbase.regionserver.port</name> + <value>16020</value> + </property> + + <property> + <name>hbase.regionserver.wal.codec</name> + <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value> + </property> + + <property> + <name>hbase.rootdir</name> + <value>hdfs://__HDP_NAMENODE__:8020/apps/hbase/data</value> + </property> + + <property> + <name>hbase.rpc.controllerfactory.class</name> + <value></value> + </property> + + <property> + <name>hbase.rpc.protection</name> + <value>authentication</value> + </property> + + <property> + <name>hbase.rpc.timeout</name> + <value>90000</value> + </property> + + <property> + <name>hbase.security.authentication</name> + <value>simple</value> + </property> + + <property> + <name>hbase.security.authorization</name> + <value>false</value> + </property> + + <property> + <name>hbase.superuser</name> + <value>hbase</value> + </property> + + <property> + <name>hbase.tmp.dir</name> + <value>/tmp/hbase-${user.name}</value> + </property> + + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value>__HDP_ZOOKEEPER_HOSTS__</value> + </property> + + <property> + <name>hbase.zookeeper.useMulti</name> + <value>true</value> + </property> + + <property> + <name>hfile.block.cache.size</name> + <value>0.40</value> + </property> + + <property> + <name>phoenix.functions.allowUserDefinedFunctions</name> + <value> </value> + </property> + + <property> + <name>phoenix.query.timeoutMs</name> + <value>60000</value> + </property> + + <property> + <name>zookeeper.session.timeout</name> + <value>90000</value> + </property> + + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase-unsecure</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml new file mode 100644 index 0000000..6367375 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml @@ -0,0 +1,281 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>dfs.block.access.token.enable</name> + <value>true</value> + </property> + + <property> + <name>dfs.blockreport.initialDelay</name> + <value>120</value> + </property> + + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit</name> + <value>true</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit.streams.cache.size</name> + <value>4096</value> + </property> + + <property> + <name>dfs.client.retry.policy.enabled</name> + <value>false</value> + </property> + + <property> + <name>dfs.cluster.administrators</name> + <value> hdfs</value> + </property> + + <property> + <name>dfs.datanode.address</name> + <value>0.0.0.0:50010</value> + </property> + + <property> + <name>dfs.datanode.balance.bandwidthPerSec</name> + <value>6250000</value> + </property> + + <property> + <name>dfs.datanode.data.dir</name> + <value>/opt/data/hdfs/dn</value> + </property> + + <property> + <name>dfs.datanode.data.dir.perm</name> + <value>750</value> + </property> + + <property> + <name>dfs.datanode.du.reserved</name> + <value>1073741824</value> + </property> + + <property> + <name>dfs.datanode.failed.volumes.tolerated</name> + <value>0</value> + </property> + + <property> + <name>dfs.datanode.http.address</name> + <value>0.0.0.0:50075</value> + </property> + + <property> + <name>dfs.datanode.https.address</name> + <value>0.0.0.0:50475</value> + </property> + + <property> + <name>dfs.datanode.ipc.address</name> + <value>0.0.0.0:8010</value> + </property> + + <property> + <name>dfs.datanode.max.transfer.threads</name> + <value>16384</value> + </property> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + </property> + + <property> + <name>dfs.encryption.key.provider.uri</name> + <value></value> + </property> + + <property> + <name>dfs.heartbeat.interval</name> + <value>3</value> + </property> + + <property> + <name>dfs.hosts.exclude</name> + <value>/etc/hadoop/conf/dfs.exclude</value> + </property> + + <property> + <name>dfs.http.policy</name> + <value>HTTP_ONLY</value> + </property> + + <property> + <name>dfs.https.port</name> + <value>50470</value> + </property> + + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/hadoop/hdfs/journalnode</value> + </property> + + <property> + <name>dfs.journalnode.http-address</name> + <value>0.0.0.0:8480</value> + </property> + + <property> + <name>dfs.journalnode.https-address</name> + <value>0.0.0.0:8481</value> + </property> + + <property> + <name>dfs.namenode.accesstime.precision</name> + <value>0</value> + </property> + + <property> + <name>dfs.namenode.audit.log.async</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.read.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.write.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.dir</name> + <value>__FS_CHECKPOINT_DIR__</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.edits.dir</name> + <value>${dfs.namenode.checkpoint.dir}</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.period</name> + <value>21600</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.txns</name> + <value>1000000</value> + </property> + + <property> + <name>dfs.namenode.fslock.fair</name> + <value>false</value> + </property> + + <property> + <name>dfs.namenode.handler.count</name> + <value>200</value> + </property> + + <property> + <name>dfs.namenode.http-address</name> + <value>__HDP_NAMENODE__:50070</value> + <final>true</final> + </property> + + <property> + <name>dfs.namenode.https-address</name> + <value>__HDP_NAMENODE__:50470</value> + </property> + + <property> + <name>dfs.namenode.name.dir</name> + <value>__DFS_NAME_DIR__</value> + </property> + + <property> + <name>dfs.namenode.name.dir.restore</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.rpc-address</name> + <value>__HDP_NAMENODE__:8020</value> + </property> + + <property> + <name>dfs.namenode.safemode.threshold-pct</name> + <value>1</value> + </property> + + <property> + <name>dfs.namenode.secondary.http-address</name> + <value>__HDP_SECONDARY_NAMENODE__:50090</value> + </property> + + <property> + <name>dfs.namenode.stale.datanode.interval</name> + <value>30000</value> + </property> + + <property> + <name>dfs.namenode.startup.delay.block.deletion.sec</name> + <value>3600</value> + </property> + + <property> + <name>dfs.namenode.write.stale.datanode.ratio</name> + <value>1.0f</value> + </property> + + <property> + <name>dfs.permissions.enabled</name> + <value>true</value> + </property> + + <property> + <name>dfs.permissions.superusergroup</name> + <value>hdfs</value> + </property> + + <property> + <name>dfs.replication</name> + <value>3</value> + </property> + + <property> + <name>dfs.replication.max</name> + <value>50</value> + </property> + + <property> + <name>dfs.support.append</name> + <value>true</value> + <final>true</final> + </property> + + <property> + <name>dfs.webhdfs.enabled</name> + <value>false</value> + </property> + + <property> + <name>fs.permissions.umask-mode</name> + <value>022</value> + </property> + + <property> + <name>nfs.exports.allowed.hosts</name> + <value>* rw</value> + </property> + + <property> + <name>nfs.file.dump.dir</name> + <value>/tmp/.hdfs-nfs</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties new file mode 100644 index 0000000..3737eac --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties @@ -0,0 +1,111 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=INFO +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers new file mode 100644 index 0000000..96dc196 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers @@ -0,0 +1,3 @@ +__DCAE_CDAP_NODE0__ +__DCAE_CDAP_NODE1__ +__DCAE_CDAP_NODE2__ diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template new file mode 100644 index 0000000..fe47d94 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, console + +######## console appender ######## +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties new file mode 100644 index 0000000..d65f9be --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties @@ -0,0 +1,5 @@ +# Generated by Apache Ambari. Thu Jun 18 00:59:57 2015 + +atlas.http.authentication.enabled=false +atlas.http.authentication.type=simple + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh new file mode 100644 index 0000000..fd99df0 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh @@ -0,0 +1,34 @@ + + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR=/usr/hdp/current/hive-client/conf + +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar +fi + +export METASTORE_PORT=9083
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template new file mode 100644 index 0000000..91a2905 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hive and Hadoop environment variables here. These variables can be used +# to control the execution of Hive. It should be used by admins to configure +# the Hive installation (so that users do not have to set environment variables +# or set command line parameters to get correct behavior). +# +# The hive service being invoked (CLI/HWI etc.) is available via the environment +# variable SERVICE + + +# Hive Client memory usage can be an issue if a large number of clients +# are running at the same time. The flags below have been useful in +# reducing memory usage: +# +# if [ "$SERVICE" = "cli" ]; then +# if [ -z "$DEBUG" ]; then +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" +# else +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" +# fi +# fi + +# The heap size of the jvm stared by hive shell script can be controlled via: +# +# export HADOOP_HEAPSIZE=1024 +# +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +# HADOOP_HOME=${bin}/../../hadoop + +# Hive Configuration Directory can be controlled by: +# export HIVE_CONF_DIR= + +# Folder containing extra ibraries required for hive compilation/execution can be controlled by: +# export HIVE_AUX_JARS_PATH= diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties new file mode 100644 index 0000000..5b59a3f --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties @@ -0,0 +1,79 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger=INFO,FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties new file mode 100644 index 0000000..7861ba5 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties @@ -0,0 +1,89 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hive.log.threshold=ALL +hive.root.logger=INFO,DRFA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.log.file=hive.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${hive.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. +# +# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,DRFA +log4j.category.Datastore=ERROR,DRFA +log4j.category.Datastore.Schema=ERROR,DRFA +log4j.category.JPOX.Datastore=ERROR,DRFA +log4j.category.JPOX.Plugin=ERROR,DRFA +log4j.category.JPOX.MetaData=ERROR,DRFA +log4j.category.JPOX.Query=ERROR,DRFA +log4j.category.JPOX.General=ERROR,DRFA +log4j.category.JPOX.Enhancer=ERROR,DRFA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml new file mode 100644 index 0000000..a3c34e3 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml @@ -0,0 +1,789 @@ +<!--Thu Jun 18 00:59:56 2015--> + <configuration> + + <property> + <name>ambari.hive.db.schema.name</name> + <value>hive</value> + </property> + + <property> + <name>datanucleus.autoCreateSchema</name> + <value>false</value> + </property> + + <property> + <name>datanucleus.cache.level2.type</name> + <value>none</value> + </property> + + <property> + <name>hive.auto.convert.join</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.join.noconditionaltask</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.join.noconditionaltask.size</name> + <value>894784504</value> + </property> + + <property> + <name>hive.auto.convert.sortmerge.join</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.sortmerge.join.to.mapjoin</name> + <value>false</value> + </property> + + <property> + <name>hive.cbo.enable</name> + <value>true</value> + </property> + + <property> + <name>hive.cli.print.header</name> + <value>false</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.class</name> + <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.zookeeper.connectString</name> + <value>TODO_REPLACE_HOSTNAME:2181</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.zookeeper.znode</name> + <value>/hive/cluster/delegation</value> + </property> + + <property> + <name>hive.cluster.name</name> + <value>companion</value> + </property> + + <property> + <name>hive.compactor.abortedtxn.threshold</name> + <value>1000</value> + </property> + + <property> + <name>hive.compactor.check.interval</name> + <value>300L</value> + </property> + + <property> + <name>hive.compactor.delta.num.threshold</name> + <value>10</value> + </property> + + <property> + <name>hive.compactor.delta.pct.threshold</name> + <value>0.1f</value> + </property> + + <property> + <name>hive.compactor.initiator.on</name> + <value>false</value> + </property> + + <property> + <name>hive.compactor.worker.threads</name> + <value>0</value> + </property> + + <property> + <name>hive.compactor.worker.timeout</name> + <value>86400L</value> + </property> + + <property> + <name>hive.compute.query.using.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.conf.restricted.list</name> + <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value> + </property> + + <property> + <name>hive.convert.join.bucket.mapjoin.tez</name> + <value>false</value> + </property> + + <property> + <name>hive.default.fileformat</name> + <value>TextFile</value> + </property> + + <property> + <name>hive.enforce.bucketing</name> + <value>false</value> + </property> + + <property> + <name>hive.enforce.sorting</name> + <value>true</value> + </property> + + <property> + <name>hive.enforce.sortmergebucketmapjoin</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.compress.intermediate</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.compress.output</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.dynamic.partition</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.dynamic.partition.mode</name> + <value>strict</value> + </property> + + <property> + <name>hive.exec.failure.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> + </property> + + <property> + <name>hive.exec.max.created.files</name> + <value>100000</value> + </property> + + <property> + <name>hive.exec.max.dynamic.partitions</name> + <value>5000</value> + </property> + + <property> + <name>hive.exec.max.dynamic.partitions.pernode</name> + <value>2000</value> + </property> + + <property> + <name>hive.exec.orc.compression.strategy</name> + <value>SPEED</value> + </property> + + <property> + <name>hive.exec.orc.default.compress</name> + <value>ZLIB</value> + </property> + + <property> + <name>hive.exec.orc.default.stripe.size</name> + <value>67108864</value> + </property> + + <property> + <name>hive.exec.orc.encoding.strategy</name> + <value>SPEED</value> + </property> + + <property> + <name>hive.exec.parallel</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.parallel.thread.number</name> + <value>8</value> + </property> + + <property> + <name>hive.exec.post.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook, org.apache.atlas.hive.hook.HiveHook</value> + </property> + + <property> + <name>hive.exec.pre.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> + </property> + + <property> + <name>hive.exec.reducers.bytes.per.reducer</name> + <value>67108864</value> + </property> + + <property> + <name>hive.exec.reducers.max</name> + <value>1009</value> + </property> + + <property> + <name>hive.exec.scratchdir</name> + <value>/tmp/hive</value> + </property> + + <property> + <name>hive.exec.submit.local.task.via.child</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.submitviachild</name> + <value>false</value> + </property> + + <property> + <name>hive.execution.engine</name> + <value>tez</value> + </property> + + <property> + <name>hive.fetch.task.aggr</name> + <value>false</value> + </property> + + <property> + <name>hive.fetch.task.conversion</name> + <value>more</value> + </property> + + <property> + <name>hive.fetch.task.conversion.threshold</name> + <value>1073741824</value> + </property> + + <property> + <name>hive.hook.dgi.url</name> + <value>http://TODO_REPLACE_HOSTNAME:21000</value> + </property> + + <property> + <name>hive.limit.optimize.enable</name> + <value>true</value> + </property> + + <property> + <name>hive.limit.pushdown.memory.usage</name> + <value>0.04</value> + </property> + + <property> + <name>hive.map.aggr</name> + <value>true</value> + </property> + + <property> + <name>hive.map.aggr.hash.force.flush.memory.threshold</name> + <value>0.9</value> + </property> + + <property> + <name>hive.map.aggr.hash.min.reduction</name> + <value>0.5</value> + </property> + + <property> + <name>hive.map.aggr.hash.percentmemory</name> + <value>0.5</value> + </property> + + <property> + <name>hive.mapjoin.bucket.cache.size</name> + <value>10000</value> + </property> + + <property> + <name>hive.mapjoin.optimized.hashtable</name> + <value>true</value> + </property> + + <property> + <name>hive.mapred.reduce.tasks.speculative.execution</name> + <value>false</value> + </property> + + <property> + <name>hive.merge.mapfiles</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.mapredfiles</name> + <value>false</value> + </property> + + <property> + <name>hive.merge.orcfile.stripe.level</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.rcfile.block.level</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.size.per.task</name> + <value>256000000</value> + </property> + + <property> + <name>hive.merge.smallfiles.avgsize</name> + <value>16000000</value> + </property> + + <property> + <name>hive.merge.tezfiles</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.authorization.storage.checks</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.cache.pinobjtypes</name> + <value>Table,Database,Type,FieldSchema,Order</value> + </property> + + <property> + <name>hive.metastore.client.connect.retry.delay</name> + <value>5s</value> + </property> + + <property> + <name>hive.metastore.client.socket.timeout</name> + <value>1800s</value> + </property> + + <property> + <name>hive.metastore.connect.retries</name> + <value>24</value> + </property> + + <property> + <name>hive.metastore.execute.setugi</name> + <value>true</value> + </property> + + <property> + <name>hive.metastore.failure.retries</name> + <value>24</value> + </property> + + <property> + <name>hive.metastore.kerberos.keytab.file</name> + <value>/etc/security/keytabs/hive.service.keytab</value> + </property> + + <property> + <name>hive.metastore.kerberos.principal</name> + <value>hive/_HOST@EXAMPLE.COM</value> + </property> + + <property> + <name>hive.metastore.pre.event.listeners</name> + <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value> + </property> + + <property> + <name>hive.metastore.sasl.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.server.max.threads</name> + <value>100000</value> + </property> + + <property> + <name>hive.metastore.uris</name> + <value>thrift://TODO_REPLACE_HOSTNAME:9083</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>/apps/hive/warehouse</value> + </property> + + <property> + <name>hive.optimize.bucketmapjoin</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.bucketmapjoin.sortedmerge</name> + <value>false</value> + </property> + + <property> + <name>hive.optimize.constant.propagation</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.index.filter</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.metadataonly</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.null.scan</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.reducededuplication</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.reducededuplication.min.reducer</name> + <value>4</value> + </property> + + <property> + <name>hive.optimize.sort.dynamic.partition</name> + <value>false</value> + </property> + + <property> + <name>hive.orc.compute.splits.num.threads</name> + <value>10</value> + </property> + + <property> + <name>hive.orc.splits.include.file.footer</name> + <value>false</value> + </property> + + <property> + <name>hive.prewarm.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.prewarm.numcontainers</name> + <value>3</value> + </property> + + <property> + <name>hive.security.authenticator.manager</name> + <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value> + </property> + + <property> + <name>hive.security.authorization.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.security.authorization.manager</name> + <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value> + </property> + + <property> + <name>hive.security.metastore.authenticator.manager</name> + <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value> + </property> + + <property> + <name>hive.security.metastore.authorization.auth.reads</name> + <value>true</value> + </property> + + <property> + <name>hive.security.metastore.authorization.manager</name> + <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value> + </property> + + <property> + <name>hive.server2.allow.user.substitution</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.authentication</name> + <value>NONE</value> + </property> + + <property> + <name>hive.server2.authentication.spnego.keytab</name> + <value>HTTP/_HOST@EXAMPLE.COM</value> + </property> + + <property> + <name>hive.server2.authentication.spnego.principal</name> + <value>/etc/security/keytabs/spnego.service.keytab</value> + </property> + + <property> + <name>hive.server2.enable.doAs</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.logging.operation.enabled</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.logging.operation.log.location</name> + <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value> + </property> + + <property> + <name>hive.server2.support.dynamic.service.discovery</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.table.type.mapping</name> + <value>CLASSIC</value> + </property> + + <property> + <name>hive.server2.tez.default.queues</name> + <value>default</value> + </property> + + <property> + <name>hive.server2.tez.initialize.default.sessions</name> + <value>false</value> + </property> + + <property> + <name>hive.server2.tez.sessions.per.default.queue</name> + <value>1</value> + </property> + + <property> + <name>hive.server2.thrift.http.path</name> + <value>cliservice</value> + </property> + + <property> + <name>hive.server2.thrift.http.port</name> + <value>10001</value> + </property> + + <property> + <name>hive.server2.thrift.max.worker.threads</name> + <value>500</value> + </property> + + <property> + <name>hive.server2.thrift.port</name> + <value>10000</value> + </property> + + <property> + <name>hive.server2.thrift.sasl.qop</name> + <value>auth</value> + </property> + + <property> + <name>hive.server2.transport.mode</name> + <value>binary</value> + </property> + + <property> + <name>hive.server2.use.SSL</name> + <value>false</value> + </property> + + <property> + <name>hive.server2.zookeeper.namespace</name> + <value>hiveserver2</value> + </property> + + <property> + <name>hive.smbjoin.cache.rows</name> + <value>10000</value> + </property> + + <property> + <name>hive.stats.autogather</name> + <value>true</value> + </property> + + <property> + <name>hive.stats.dbclass</name> + <value>fs</value> + </property> + + <property> + <name>hive.stats.fetch.column.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.stats.fetch.partition.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.support.concurrency</name> + <value>false</value> + </property> + + <property> + <name>hive.tez.auto.reducer.parallelism</name> + <value>true</value> + </property> + + <property> + <name>hive.tez.container.size</name> + <value>2560</value> + </property> + + <property> + <name>hive.tez.cpu.vcores</name> + <value>-1</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning</name> + <value>true</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning.max.data.size</name> + <value>104857600</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning.max.event.size</name> + <value>1048576</value> + </property> + + <property> + <name>hive.tez.input.format</name> + <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value> + </property> + + <property> + <name>hive.tez.java.opts</name> + <value>-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps</value> + </property> + + <property> + <name>hive.tez.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>hive.tez.max.partition.factor</name> + <value>2.0</value> + </property> + + <property> + <name>hive.tez.min.partition.factor</name> + <value>0.25</value> + </property> + + <property> + <name>hive.tez.smb.number.waves</name> + <value>0.5</value> + </property> + + <property> + <name>hive.txn.manager</name> + <value>org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager</value> + </property> + + <property> + <name>hive.txn.max.open.batch</name> + <value>1000</value> + </property> + + <property> + <name>hive.txn.timeout</name> + <value>300</value> + </property> + + <property> + <name>hive.user.install.directory</name> + <value>/user/</value> + </property> + + <property> + <name>hive.vectorized.execution.enabled</name> + <value>true</value> + </property> + + <property> + <name>hive.vectorized.execution.reduce.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.vectorized.groupby.checkinterval</name> + <value>4096</value> + </property> + + <property> + <name>hive.vectorized.groupby.flush.percent</name> + <value>0.1</value> + </property> + + <property> + <name>hive.vectorized.groupby.maxentries</name> + <value>100000</value> + </property> + + <property> + <name>hive.zookeeper.client.port</name> + <value>2181</value> + </property> + + <property> + <name>hive.zookeeper.namespace</name> + <value>hive_zookeeper_namespace</value> + </property> + + <property> + <name>hive.zookeeper.quorum</name> + <value>TODO_REPLACE_HOSTNAME:2181</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionDriverName</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:mysql://TODO_REPLACE_HOSTNAME/hive?createDatabaseIfNotExist=true</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionUserName</name> + <value>hive</value> + </property> + + <property> + <name>mapreduce.input.fileinputformat.split.minsize</name> + <value>1000000</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml new file mode 100644 index 0000000..bda842a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml @@ -0,0 +1,37 @@ + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!--This file is used by grapes to download dependencies from a maven repository. + This is just a template and can be edited to add more repositories. +--> + +<ivysettings> + <!--name of the defaultResolver should always be 'downloadGrapes'. --> + <settings defaultResolver="downloadGrapes"/> + <resolvers> + <!-- more resolvers can be added here --> + <chain name="downloadGrapes"> + <!-- This resolver uses ibiblio to find artifacts, compatible with maven2 repository --> + <ibiblio name="central" m2compatible="true"/> + <!-- File resolver to add jars from the local system. --> + <filesystem name="test" checkmodified="true"> + <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar" /> + </filesystem> + </chain> + </resolvers> +</ivysettings> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml new file mode 100644 index 0000000..6c71cae --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml @@ -0,0 +1,224 @@ +<!--Thu Jun 18 04:07:38 2015--> + <configuration> + + <property> + <name>mapreduce.admin.map.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.reduce.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.user.env</name> + <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + </property> + + <property> + <name>mapreduce.am.max-attempts</name> + <value>2</value> + </property> + + <property> + <name>mapreduce.application.classpath</name> + <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value> + </property> + + <property> + <name>mapreduce.application.framework.path</name> + <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value> + </property> + + <property> + <name>mapreduce.cluster.administrators</name> + <value> hadoop</value> + </property> + + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + + <property> + <name>mapreduce.job.counters.max</name> + <value>130</value> + </property> + + <property> + <name>mapreduce.job.emit-timeline-data</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.job.reduce.slowstart.completedmaps</name> + <value>0.05</value> + </property> + + <property> + <name>mapreduce.jobhistory.address</name> + <value>TODO_REPLACE_HOSTNAME:10020</value> + </property> + + <property> + <name>mapreduce.jobhistory.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>mapreduce.jobhistory.done-dir</name> + <value>/mr-history/done</value> + </property> + + <property> + <name>mapreduce.jobhistory.intermediate-done-dir</name> + <value>/mr-history/tmp</value> + </property> + + <property> + <name>mapreduce.jobhistory.webapp.address</name> + <value>TODO_REPLACE_HOSTNAME:19888</value> + </property> + + <property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx2048m</value> + </property> + + <property> + <name>mapreduce.map.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.map.memory.mb</name> + <value>2560</value> + </property> + + <property> + <name>mapreduce.map.output.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.map.sort.spill.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.map.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress.type</name> + <value>BLOCK</value> + </property> + + <property> + <name>mapreduce.reduce.input.buffer.percent</name> + <value>0.0</value> + </property> + + <property> + <name>mapreduce.reduce.java.opts</name> + <value>-Xmx4096m</value> + </property> + + <property> + <name>mapreduce.reduce.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.reduce.memory.mb</name> + <value>5120</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name> + <value>1</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name> + <value>1000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name> + <value>30000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.input.buffer.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.merge.percent</name> + <value>0.66</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.parallelcopies</name> + <value>30</value> + </property> + + <property> + <name>mapreduce.reduce.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.shuffle.port</name> + <value>13562</value> + </property> + + <property> + <name>mapreduce.task.io.sort.factor</name> + <value>100</value> + </property> + + <property> + <name>mapreduce.task.io.sort.mb</name> + <value>1433</value> + </property> + + <property> + <name>mapreduce.task.timeout</name> + <value>300000</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.admin-command-opts</name> + <value>-Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.command-opts</name> + <value>-Xmx2048m -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.resource.mb</name> + <value>2560</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.staging-dir</name> + <value>/user</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties new file mode 100644 index 0000000..83847de --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# Zookeeper connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zookeeper.connect=127.0.0.1:2181 + +# timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=6000 + +#consumer group id +group.id=test-consumer-group + +#consumer timeout +#consumer.timeout.ms=5000 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh new file mode 100644 index 0000000..cbfc937 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh @@ -0,0 +1,19 @@ + +#!/bin/bash + +# Set KAFKA specific environment variables here. + +# The java implementation to use. +export JAVA_HOME=/usr/jdk64/jdk1.8.0_40 +export PATH=$PATH:$JAVA_HOME/bin +export PID_DIR=/var/run/kafka +export LOG_DIR=/var/log/kafka +export KAFKA_KERBEROS_PARAMS= +# Add kafka sink to classpath and related depenencies +if [ -e "/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar" ]; then + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/* +fi +if [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then +. /etc/kafka/conf/kafka-ranger-env.sh +fi diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties new file mode 100644 index 0000000..2cba04d --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties @@ -0,0 +1,84 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# +kafka.logs.dir=logs + +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +#log4j.logger.kafka.perf=DEBUG, kafkaAppender +#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties new file mode 100644 index 0000000..47ae3e2 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.producer.ProducerConfig for more details + +############################# Producer Basics ############################# + +# list of brokers used for bootstrapping knowledge about the rest of the cluster +# format: host1:port1,host2:port2 ... +metadata.broker.list=localhost:9092 + +# name of the partitioner class for partitioning events; default partition spreads data randomly +#partitioner.class= + +# specifies whether the messages are sent asynchronously (async) or synchronously (sync) +producer.type=sync + +# specify the compression codec for all data generated: none, gzip, snappy, lz4. +# the old config values work as well: 0, 1, 2, 3 for none, gzip, snappy, lz4, respectively +compression.codec=none + +# message encoder +serializer.class=kafka.serializer.DefaultEncoder + +# allow topic level compression +#compressed.topics= + +############################# Async Producer ############################# +# maximum time, in milliseconds, for buffering data on the producer queue +#queue.buffering.max.ms= + +# the maximum size of the blocking queue for buffering on the producer +#queue.buffering.max.messages= + +# Timeout for event enqueue: +# 0: events will be enqueued immediately or dropped if the queue is full +# -ve: enqueue will block indefinitely if the queue is full +# +ve: enqueue will block up to this many milliseconds if the queue is full +#queue.enqueue.timeout.ms= + +# the number of messages batched at the producer +#batch.num.messages= diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties new file mode 100644 index 0000000..7dfb879 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties @@ -0,0 +1,70 @@ +auto.leader.rebalance.enable=true +kafka.ganglia.metrics.port=8671 +socket.send.buffer.bytes=102400 +message.max.bytes=1000000 +num.network.threads=3 +log.segment.bytes=1073741824 +kafka.ganglia.metrics.host=localhost +leader.imbalance.per.broker.percentage=10 +replica.lag.time.max.ms=10000 +authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer +num.io.threads=8 +offsets.retention.minutes=86400000 +fetch.purgatory.purge.interval.requests=10000 +offsets.topic.compression.codec=0 +replica.lag.max.messages=4000 +num.recovery.threads.per.data.dir=1 +log.dirs=/grid/0/kafka-logs +log.retention.bytes=-1 +producer.purgatory.purge.interval.requests=10000 +log.flush.scheduler.interval.ms=3000 +default.replication.factor=1 +replica.high.watermark.checkpoint.interval.ms=5000 +zookeeper.connect=TODO_REPLACE_HOSTNAME:2181 +controlled.shutdown.retry.backoff.ms=5000 +kafka.timeline.metrics.host=TODO_REPLACE_HOSTNAME +kafka.ganglia.metrics.group=kafka +kafka.timeline.metrics.reporter.sendInterval=5900 +num.partitions=1 +offsets.topic.segment.bytes=104857600 +controller.socket.timeout.ms=30000 +queued.max.requests=500 +broker.id=0 +offset.metadata.max.bytes=4096 +kafka.timeline.metrics.reporter.enabled=true +controlled.shutdown.max.retries=3 +min.insync.replicas=1 +offsets.commit.required.acks=-1 +replica.fetch.wait.max.ms=500 +controlled.shutdown.enable=true +log.roll.hours=168 +log.cleanup.interval.mins=10 +replica.socket.receive.buffer.bytes=65536 +kafka.ganglia.metrics.reporter.enabled=true +offsets.topic.num.partitions=50 +delete.topic.enable=false +offsets.load.buffer.size=5242880 +num.replica.fetchers=1 +socket.request.max.bytes=104857600 +kafka.timeline.metrics.maxRowCacheSize=10000 +kafka.timeline.metrics.port=6188 +compression.type=producer +zookeeper.sync.time.ms=2000 +zookeeper.connection.timeout.ms=6000 +socket.receive.buffer.bytes=102400 +controller.message.queue.size=10 +offsets.commit.timeout.ms=5000 +offsets.topic.replication.factor=3 +offsets.retention.check.interval.ms=600000 +log.flush.interval.ms=3000 +replica.fetch.max.bytes=1048576 +kafka.metrics.reporters=org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter +listeners=PLAINTEXT://TODO_REPLACE_HOSTNAME:6667 +replica.fetch.min.bytes=1 +replica.socket.timeout.ms=30000 +zookeeper.session.timeout.ms=30000 +auto.create.topics.enable=true +log.index.size.max.bytes=10485760 +leader.imbalance.check.interval.seconds=300 +log.index.interval.bytes=4096 +log.retention.hours=168 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties new file mode 100644 index 0000000..e0bbc13 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=logs/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=logs/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=logs/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=logs/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +log4j.logger.kafka.tools=DEBUG, kafkaAppender +log4j.logger.kafka.tools.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=TRACE, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties new file mode 100644 index 0000000..52f07c9 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties new file mode 100644 index 0000000..74cbf90 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# disable the per-ip limit on the number of connections since this is a non-production config +maxClientCnxns=0 diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl new file mode 100644 index 0000000..3e0216a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl @@ -0,0 +1,25 @@ + +<?xml version="1.0"?> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"> +<xsl:output method="html"/> +<xsl:template match="configuration"> +<html> +<body> +<table border="1"> +<tr> + <td>name</td> + <td>value</td> + <td>description</td> +</tr> +<xsl:for-each select="property"> + <tr> + <td><a name="{name}"><xsl:value-of select="name"/></a></td> + <td><xsl:value-of select="value"/></td> + <td><xsl:value-of select="description"/></td> + </tr> +</xsl:for-each> +</table> +</body> +</html> +</xsl:template> +</xsl:stylesheet> diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties new file mode 100644 index 0000000..20e1beb --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties @@ -0,0 +1,70 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# +# ZooKeeper Logging Configuration +# + +# DEFAULT: console appender only +log4j.rootLogger=INFO, CONSOLE + +# Example with rolling log file +#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=DEBUG +log4j.appender.ROLLINGFILE.File=zookeeper.log + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=zookeeper_trace.log + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg new file mode 100644 index 0000000..3d9911b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg @@ -0,0 +1,37 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +clientPort=2181 +initLimit=10 +autopurge.purgeInterval=24 +syncLimit=5 +tickTime=2000 +dataDir=__ZOOKEEPER_DATA_DIR__ +autopurge.snapRetainCount=30 +server.0=__DCAE_CDAP_NODE0__:2888:3888 +server.1=__DCAE_CDAP_NODE1__:2888:3888 +server.2=__DCAE_CDAP_NODE2__:2888:3888 + +# [DE239909] 3 Oct 2016 +maxClientCnxns=0 + diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg new file mode 100644 index 0000000..a5a2c0b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg @@ -0,0 +1,28 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +#autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +#autopurge.purgeInterval=1 diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd new file mode 100644 index 0000000..bad3b7a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd @@ -0,0 +1,28 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@rem Set environment variables here. + +@rem JVM parameters for both client and server +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +set JVMFLAGS=-Djava.net.preferIPv4Stack=true + +@rem Client specific JVM parameters +@rem set CLIENT_JVMFLAGS= + +@rem Server specific JVM parameters +@rem set SERVER_JVMFLAGS= diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh new file mode 100644 index 0000000..8a545b3 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh @@ -0,0 +1,9 @@ + +export JAVA_HOME=__JAVA_HOME__ +export ZOOKEEPER_HOME=/usr/hdp/current/zookeeper-server +export ZOO_LOG_DIR=__ZOOKEEPER_LOG_DIR__ +export ZOOPIDFILE=__ZOOKEEPER_PID_DIR__/zookeeper_server.pid +export SERVER_JVMFLAGS=-Xmx1024m +export JAVA=$JAVA_HOME/bin/java +export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* + diff --git a/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl b/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl new file mode 100644 index 0000000..773f6ee --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl @@ -0,0 +1,11 @@ +__HDP_NAMENODE__=__DCAE_CDAP_NODE0__ +__HDP_HBASE_MASTER__=__DCAE_CDAP_NODE1__ +__HDP_APP_TIMELINE_SERVER__=__DCAE_CDAP_NODE1__ +__HDP_HISTORYSERVER__=__DCAE_CDAP_NODE1__ +__HDP_RESOURCEMANAGER__=__DCAE_CDAP_NODE1__ +__HDP_SECONDARY_NAMENODE__=__DCAE_CDAP_NODE1__ +__HDP_ZOOKEEPER_HOSTS_WITH_PORT__=__DCAE_CDAP_NODE0__:2181,__DCAE_CDAP_NODE1__:2181,__DCAE_CDAP_NODE2__:2181 +__HDP_ZOOKEEPER_QUORUM__=__DCAE_CDAP_NODE0__,__DCAE_CDAP_NODE1__,__DCAE_CDAP_NODE2__:2181 +__HDP_ZOOKEEPER_HOSTS__=__DCAE_CDAP_NODE0__,__DCAE_CDAP_NODE1__,__DCAE_CDAP_NODE2__ +__CDAP_ROUTER_HOST__=__DCAE_CDAP_NODE2__ +__CDAP_KAFKA_HOST__=__DCAE_CDAP_NODE2__ diff --git a/cdap3vm/config/hostname-xref.txt.test b/cdap3vm/config/hostname-xref.txt.test new file mode 100644 index 0000000..faac169 --- /dev/null +++ b/cdap3vm/config/hostname-xref.txt.test @@ -0,0 +1,3 @@ +__DCAE_CDAP_NODE0__=c3p1.novalocal +__DCAE_CDAP_NODE1__=c3p0.novalocal +__DCAE_CDAP_NODE2__=c3p2.novalocal diff --git a/cdap3vm/config/services-setup-templates/cdap-setup.sh b/cdap3vm/config/services-setup-templates/cdap-setup.sh new file mode 100644 index 0000000..32c0891 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/cdap-setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Set up files and folders + +# Start CDAP services + diff --git a/cdap3vm/config/services-setup-templates/cdap-start.sh b/cdap3vm/config/services-setup-templates/cdap-start.sh new file mode 100644 index 0000000..5b7965b --- /dev/null +++ b/cdap3vm/config/services-setup-templates/cdap-start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export JAVA_HOME=__JAVA_HOME__ + +/etc/init.d/cdap-auth-server start +/etc/init.d/cdap-kafka-server start +/etc/init.d/cdap-master start +/etc/init.d/cdap-router start + +PATH=$PATH:__NODEJS_BIN__ +/etc/init.d/cdap-ui start diff --git a/cdap3vm/config/services-setup-templates/datanode-start.sh b/cdap3vm/config/services-setup-templates/datanode-start.sh new file mode 100644 index 0000000..c0d2076 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/datanode-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start datanode + diff --git a/cdap3vm/config/services-setup-templates/hbase-master-start.sh b/cdap3vm/config/services-setup-templates/hbase-master-start.sh new file mode 100644 index 0000000..62329cf --- /dev/null +++ b/cdap3vm/config/services-setup-templates/hbase-master-start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh b/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh new file mode 100644 index 0000000..08eee7c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +sleep 100 # Wait for all things to sync up? +__HDP_CURRENT_FOLDER__/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh b/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh new file mode 100644 index 0000000..d08237c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +chown -R __SUPER_USER__:__HADOOP_USER__ __HDP_CURRENT_FOLDER__/hadoop-yarn*/bin/container-executor +chmod -R 6050 __HDP_CURRENT_FOLDER__/hadoop-yarn*/bin/container-executor diff --git a/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh b/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh new file mode 100644 index 0000000..5671c35 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +hdfs dfs -mkdir -p /mr-history/tmp +hdfs dfs -chmod -R 1777 /mr-history/tmp + +hdfs dfs -mkdir -p /mr-history/done +hdfs dfs -chmod -R 1777 /mr-history/done +hdfs dfs -chown -R __MAPRED_USER__:__HDFS_USER__ /mr-history + +hdfs dfs -mkdir -p /app-logs +hdfs dfs -chmod -R 1777 /app-logs + +hdfs dfs -chown __YARN_USER__:__HDFS_USER__ /app-logs + + + diff --git a/cdap3vm/config/services-setup-templates/namenode-setup.sh b/cdap3vm/config/services-setup-templates/namenode-setup.sh new file mode 100644 index 0000000..5ba1297 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/namenode-setup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +__HDP_HADOOP_BIN__/hdfs namenode -format -force +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start namenode +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start datanode + +# Create HBASE related folders here +( +hdfs dfs -mkdir -p /apps/hbase/staging /apps/hbase/data + +hdfs dfs -chown hbase:hdfs /apps/hbase/staging /apps/hbase/data +hdfs dfs -chmod 711 /apps/hbase/staging +hdfs dfs -chmod 755 /apps/hbase/data + +hdfs dfs -chown hdfs:hdfs /apps/hbase +) & + +hdfs dfs -mkdir -p /hdp/apps/__HDP_VERSION__/mapreduce/ +hdfs dfs -put __HDP_CURRENT_FOLDER__/hadoop-client/mapreduce.tar.gz /hdp/apps/__HDP_VERSION__/mapreduce/ +hdfs dfs -chown -R __HDFS_USER__:__HADOOP_GROUP__ /hdp +hdfs dfs -chmod -R 555 /hdp/apps/__HDP_VERSION__/mapreduce +hdfs dfs -chmod 444 /hdp/apps/__HDP_VERSION__/mapreduce/mapreduce.tar.gz + +# We will try to set up general CDAP related stuff (cluster-prep) here +( +hdfs dfs -mkdir -p /user/yarn +hdfs dfs -chown yarn:yarn /user/yarn + +hdfs dfs -mkdir -p /cdap +hdfs dfs -chown yarn /cdap + +hdfs dfs -mkdir -p /cdap/tx.snapshot +hdfs dfs -chown yarn /cdap/tx.snapshot +) & diff --git a/cdap3vm/config/services-setup-templates/node-manager-start.sh b/cdap3vm/config/services-setup-templates/node-manager-start.sh new file mode 100644 index 0000000..6ebd914 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/node-manager-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ start nodemanager + diff --git a/cdap3vm/config/services-setup-templates/resource-manager-start.sh b/cdap3vm/config/services-setup-templates/resource-manager-start.sh new file mode 100644 index 0000000..85c660c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/resource-manager-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ start resourcemanager + diff --git a/cdap3vm/config/services-setup-templates/service-setup.sh b/cdap3vm/config/services-setup-templates/service-setup.sh new file mode 100644 index 0000000..a331c61 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/service-setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +MAINLOC=$(dirname $0) +setup_script="${1}" + +if [ -z "$2" ]; then + bash ${MAINLOC}/${setup_script} +else + su "$2" -c "bash ${MAINLOC}/${setup_script}" +fi diff --git a/cdap3vm/config/services-setup-templates/zookeeper-start.sh b/cdap3vm/config/services-setup-templates/zookeeper-start.sh new file mode 100644 index 0000000..2f974f1 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/zookeeper-start.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# ZooKeeper + +ACTION="start" + +export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__ +export ZOOCFG=zoo.cfg +source ${ZOOCFGDIR}/zookeeper-env.sh +__HDP_CURRENT_FOLDER__/zookeeper-server/bin/zkServer.sh "$ACTION" + + diff --git a/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh b/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh new file mode 100644 index 0000000..319fff3 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh @@ -0,0 +1,3 @@ +#!/bash + +__HDP_HADOOP_SBIN__/hadoop-daemon.sh start zkfc diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh new file mode 100644 index 0000000..c5c04ca --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + # [DE248720] start namenode on 00 + bash $MAINDIR/namenode.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + + wait_for_hbase_shell_OK + bash $MAINDIR/hbase-regionserver.sh $ACTION ;; + stop ) + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + # [DE256148] stop namenode on 00 + bash $MAINDIR/namenode.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh new file mode 100644 index 0000000..83145ad --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/resource-manager.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/job-history-server.sh $ACTION + + wait_for_namenode + bash $MAINDIR/hbase-master.sh $ACTION + bash $MAINDIR/hbase-regionserver.sh $ACTION ;; + stop ) + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/hbase-master.sh $ACTION + + bash $MAINDIR/job-history-server.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/resource-manager.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh new file mode 100644 index 0000000..7eebfa0 --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + + wait_for_hbase_shell_OK + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/cdap.sh $ACTION ;; + stop ) + bash $MAINDIR/cdap.sh $ACTION + bash $MAINDIR/hbase-regionserver.sh $ACTION + + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac diff --git a/cdap3vm/config/services-templates/cdap-vm-services b/cdap3vm/config/services-templates/cdap-vm-services new file mode 100644 index 0000000..832847d --- /dev/null +++ b/cdap3vm/config/services-templates/cdap-vm-services @@ -0,0 +1,46 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# description: Starts and stops all CDAP related services +# +### BEGIN INIT INFO +# Provides: cdap-vm-services +# Short-Description: Hadoop and CDAP services for this CDAP node +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +ACTION="$1" + +case "$ACTION" in + start|stop ) + bash $MAINDIR/boot-time-cdap-vm.sh "$ACTION" + ( cd /opt/app/dcae-controller-service-cdap-cluster-manager ; sudo su dcae -c "JAVA_HOME=/opt/app/java/jdk/jdk170 bin/manager.sh $ACTION" ) ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/cdap.sh b/cdap3vm/config/services-templates/cdap.sh new file mode 100644 index 0000000..24c79df --- /dev/null +++ b/cdap3vm/config/services-templates/cdap.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# CDAP Services + +ACTION="$1" + +case "$ACTION" in + start|stop|status ) + /etc/init.d/cdap-auth-server $ACTION + /etc/init.d/cdap-kafka-server $ACTION + /etc/init.d/cdap-master $ACTION + /etc/init.d/cdap-router $ACTION + /etc/init.d/cdap-ui $ACTION ;; + * ) + echo "Usage: $0 <start|stop|status>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/datanode.sh b/cdap3vm/config/services-templates/datanode.sh new file mode 100644 index 0000000..29cbcc9 --- /dev/null +++ b/cdap3vm/config/services-templates/datanode.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Datanode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION datanode" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + + + diff --git a/cdap3vm/config/services-templates/hbase-master.sh b/cdap3vm/config/services-templates/hbase-master.sh new file mode 100644 index 0000000..f928a0d --- /dev/null +++ b/cdap3vm/config/services-templates/hbase-master.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# HBase Master + +ACTION="$1" + +case "$ACTION" in + start|stop|restart ) + su - "__HBASE_USER__" -c "__HDP_CURRENT_FOLDER__/hbase-master/bin/hbase-daemon.sh $ACTION master" ;; + * ) + echo "Usage: $0 <start|stop|restart>" + exit -1 ;; +esac + + + diff --git a/cdap3vm/config/services-templates/hbase-regionserver.sh b/cdap3vm/config/services-templates/hbase-regionserver.sh new file mode 100644 index 0000000..83bba40 --- /dev/null +++ b/cdap3vm/config/services-templates/hbase-regionserver.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# HBase Region Server + +ACTION="$1" + +case "$ACTION" in + start|stop|restart ) + su - "__HBASE_USER__" -c "__HDP_CURRENT_FOLDER__/hbase-regionserver/bin/hbase-daemon.sh $ACTION regionserver" ;; + * ) + echo "Usage: $0 <start|stop|restart>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/job-history-server.sh b/cdap3vm/config/services-templates/job-history-server.sh new file mode 100644 index 0000000..8f39aba --- /dev/null +++ b/cdap3vm/config/services-templates/job-history-server.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Job History Server + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su "__HDFS_USER__" -c "__HDP_CURRENT_FOLDER__/hadoop-mapreduce-historyserver/sbin/mr-jobhistory-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION historyserver" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + + + + diff --git a/cdap3vm/config/services-templates/namenode.sh b/cdap3vm/config/services-templates/namenode.sh new file mode 100644 index 0000000..fac7bac --- /dev/null +++ b/cdap3vm/config/services-templates/namenode.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Namenode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION namenode" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/node-manager.sh b/cdap3vm/config/services-templates/node-manager.sh new file mode 100644 index 0000000..e2fd187 --- /dev/null +++ b/cdap3vm/config/services-templates/node-manager.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Yarn Node Manager + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __YARN_USER__ -c "__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION nodemanager" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/resource-manager.sh b/cdap3vm/config/services-templates/resource-manager.sh new file mode 100644 index 0000000..e5b1a9f --- /dev/null +++ b/cdap3vm/config/services-templates/resource-manager.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Yarn Resource Manager + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __YARN_USER__ -c "__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION resourcemanager" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + + + + diff --git a/cdap3vm/config/services-templates/secondary-namenode.sh b/cdap3vm/config/services-templates/secondary-namenode.sh new file mode 100644 index 0000000..7fd5c3f --- /dev/null +++ b/cdap3vm/config/services-templates/secondary-namenode.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Secondary Namenode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION secondarynamenode" ;; + * ) + echo "Usage: $0 <start|stop>" + exit -1 ;; +esac + + diff --git a/cdap3vm/config/services-templates/service-start.sh b/cdap3vm/config/services-templates/service-start.sh new file mode 100644 index 0000000..a3094f9 --- /dev/null +++ b/cdap3vm/config/services-templates/service-start.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +service_name="${1%.sh}" + +username=${2:-$service_name} + +# Usage: $0 zookeeper will result in su zookeeper -c +su "$username" -c "bash ${MAINLOC}/${service_name}.sh start" diff --git a/cdap3vm/config/services-templates/utility-scripts.sh b/cdap3vm/config/services-templates/utility-scripts.sh new file mode 100644 index 0000000..5bfbe86 --- /dev/null +++ b/cdap3vm/config/services-templates/utility-scripts.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +wait_for_remote_service() { + remote_host="$1" + remote_port="$2" + sleep_time=${3:-30} # default of 30 seconds between retries + + # keep checking remote_host's remote_port every sleep_time seconds till we get a connection + while ( ! nc $remote_host $remote_port < /dev/null ); do sleep $sleep_time ; done +} + +wait_for_namenode() { + # keep checking namenode's port 8020 till it is up -- do it every 30 seconds + wait_for_remote_service __HDP_NAMENODE__ 8020 30 +} + +wait_for_hbase_shell_OK() { + # run hbase shell and see if we connect to hbase... Better than waiting for ports + while ( echo list | hbase shell 2>&1 | grep ^ERROR > /dev/null ); do + sleep 30 + done +} + diff --git a/cdap3vm/config/services-templates/zookeeper-zkfc.sh b/cdap3vm/config/services-templates/zookeeper-zkfc.sh new file mode 100644 index 0000000..d32973c --- /dev/null +++ b/cdap3vm/config/services-templates/zookeeper-zkfc.sh @@ -0,0 +1,33 @@ +#!/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper ZKFC + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su "__HDFS_USER__" -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh $ACTION zkfc" ;; + * ) + echo "Usage: $0 <start|stop|restart|status>" + exit -1 ;; +esac + + diff --git a/cdap3vm/config/services-templates/zookeeper.sh b/cdap3vm/config/services-templates/zookeeper.sh new file mode 100644 index 0000000..666f40c --- /dev/null +++ b/cdap3vm/config/services-templates/zookeeper.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="$1" + +export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__ + +case "$ACTION" in + start|stop|restart|status ) + su "__ZOOKEEPER_USER__" -c "export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__; export ZOOCFG=zoo.cfg; source ${ZOOCFGDIR}/zookeeper-env.sh; __HDP_CURRENT_FOLDER__/zookeeper-server/bin/zkServer.sh $ACTION" ;; + * ) + echo "Usage: $0 <start|stop|restart|status>" + exit -1 ;; +esac + + diff --git a/cdap3vm/install-steps/01-generate-host-ids-configs.sh b/cdap3vm/install-steps/01-generate-host-ids-configs.sh new file mode 100644 index 0000000..3a53078 --- /dev/null +++ b/cdap3vm/install-steps/01-generate-host-ids-configs.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +cd $(dirname $0) + +CFG=../config +HOST_XREF=${CFG}/hostname-xref.txt +CLUSTER_H_TEMPLATE=${CFG}/hadoop-cluster-hosts-file.sh.tmpl +CLUSTER_HOSTS=${CFG}/hadoop-cluster-hosts-file.sh + +HADOOP_CONF_FOLDER=../config +HADOOP_SCRIPT_FOLDER=../pkgs/services +HADOOP_SERVICE_SETUP_FOLDER=../pkgs/services-setup + +TEMPL_CDAP_CONF=../config/cdap-config-template +CDAP_CONF_FOLDER=../config/cdap-config + +# Hack to set system time to UTC first time +if [ -e /etc/localtime ]; then + rm /etc/localtime # remove it even if it is a symbolic link +fi +cp /usr/share/zoneinfo/UTC /etc/localtime +hwclock --systohc + +# Hack for 127.0.1.1 entry in /etc/hosts on Ubuntu images +# +sed '/^#/! {/127\.0\.1\.1/ s/^/# /}' -i /etc/hosts + +if [ "--no-hosts" = "$1" ]; then + echo "Using test hosts" + HOST_XREF=${HOST_XREF}.test +else + bash ./utils/generate-hosts.sh > ${HOST_XREF} +fi + +process_template() { + template="$1" + output_file="$2" + xref_file="$3" + python ./utils/replace-kv.py ${template} ${output_file} ${xref_file} +} + +process_hadoop_service_scripts() { + for x in "$@"; do + srcscript=${HADOOP_CONF_FOLDER}/services-templates/"${x}" + destscript=${HADOOP_SCRIPT_FOLDER}/"${x}" + xref_file=${MAIN_CLUSTER_CONFIG_XREF} + process_template ${srcscript} ${destscript} ${xref_file} + done +} + +process_hadoop_service_setup_scripts() { + for x in "$@"; do + srcscript=${HADOOP_CONF_FOLDER}/services-setup-templates/"${x}" + destscript=${HADOOP_SERVICE_SETUP_FOLDER}/"${x}" + xref_file=${MAIN_CLUSTER_CONFIG_XREF} + process_template ${srcscript} ${destscript} ${xref_file} + done +} + + +process_template ${CLUSTER_H_TEMPLATE} ${CLUSTER_HOSTS} ${HOST_XREF} + +# CLUSTER_HOSTS now has information on what NODE0, NODE1, etc., mean +# CLUSTER_HADOOP_CONF has information on folders, etc. +CLUSTER_HADOOP_CONF=${CFG}/hadoop-cluster-conf-file.sh + +MAIN_CLUSTER_CONFIG_XREF=${CFG}/main-conf.sh +# group them together +cat ${CLUSTER_HADOOP_CONF} ${CLUSTER_HOSTS} ${HOST_XREF} > ${MAIN_CLUSTER_CONFIG_XREF} + +# Create target configs from hadoop-config-templates +HCFG_TEMPL=${CFG}/hadoop-cluster-config-template + +HCFG=${CFG}/hadoop-cluster-config + +process_config_files() { + sfolder="$1" + mkdir -p ${HCFG}/${sfolder} + for x in ${HCFG_TEMPL}/${sfolder}/*; do + if [ -d $x ]; then continue; fi # skip folder + item=${x#$HCFG_TEMPL/} + template=${HCFG_TEMPL}/${item} + output_file=${HCFG}/${item} + process_template ${template} ${output_file} ${MAIN_CLUSTER_CONFIG_XREF} + done +} + +process_config_folder() { + sfolder="$1" + dfolder="$2" + mkdir -p ${dfolder} + for x in ${sfolder}/*; do + if [ -d $x ]; then continue; fi # skip folder + item=${x#$sfolder/} + template=${x} + output_file=${dfolder}/${item} + process_template ${template} ${output_file} ${MAIN_CLUSTER_CONFIG_XREF} + done +} + +process_config_files core_hadoop +process_config_files zookeeper +process_config_files hbase + +process_config_folder ${TEMPL_CDAP_CONF} ${CDAP_CONF_FOLDER} +process_config_folder ${TEMPL_CDAP_CONF}/common ${CDAP_CONF_FOLDER}/common + + +# TODO: We can simply process all files in the template folder for service_scripts? +process_hadoop_service_scripts \ + service-start.sh zookeeper.sh resource-manager.sh zookeeper-zkfc.sh \ + node-manager.sh namenode.sh datanode.sh secondary-namenode.sh \ + job-history-server.sh hbase-master.sh hbase-regionserver.sh \ + utility-scripts.sh cdap.sh cdap-vm-services \ + boot-time-cdap-vm-N0.sh boot-time-cdap-vm-N1.sh boot-time-cdap-vm-N2.sh + + +# TODO: We can simply process all files in the template folder for service_setup_scripts? +process_hadoop_service_setup_scripts \ + service-setup.sh namenode-setup.sh zookeeper-start.sh zookeeper-zkfc-start.sh \ + resource-manager-start.sh node-manager-start.sh datanode-start.sh \ + job-history-setup-01-as-root.sh job-history-setup-02-as-hdfs.sh \ + datanode-start.sh hbase-master-start.sh hbase-regionserver-start.sh \ + cdap-setup.sh cdap-start.sh + +chmod -R o+r ${HADOOP_SCRIPT_FOLDER}/.. + diff --git a/cdap3vm/install-steps/02-user-creation.sh b/cdap3vm/install-steps/02-user-creation.sh new file mode 100644 index 0000000..b5d6f03 --- /dev/null +++ b/cdap3vm/install-steps/02-user-creation.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Create user accounts for CDAP-Hadoop (3VM minimal) solution +# one argument after DESC creates a user x1:x1 +# additional "other-groups" argument creates a user x1:x1 -G y1,y2,... +# additional groups have to be specified in a comma-separated format without spaces + +while read desc user othergrouplist; do + if [ -z "${othergrouplist}" ]; then + useradd $user -m + else + useradd $user -m + usermod -a -G ${othergrouplist} $user + fi +done << __EOF__ +HADOOP hadoop +HDFS hdfs hadoop +HBase hbase hadoop +YARN yarn hadoop +MapReduce mapred hadoop +ZooKeeper zookeeper hadoop +CDAP cdap +DCAE dcae hdfs +__EOF__ + diff --git a/cdap3vm/install-steps/04-folder-creation.sh b/cdap3vm/install-steps/04-folder-creation.sh new file mode 100644 index 0000000..59ca5cf --- /dev/null +++ b/cdap3vm/install-steps/04-folder-creation.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +\. ../config/main-conf.sh +\. ./utils/folder-creation-utils.sh + +HADOOP_SERVICE_SETUP_FOLDER=../pkgs/services-setup + +NODETYPE=N$(get_cdap_host_id) +NODETYPE=${1:-$NODETYPE} + +X=$NODETYPE +if [[ "$X" != "N0" && "$X" != "N1" && "$X" != "N2" && "$X" != "NTEST" ]]; then + exit -1 +fi + +if [ "$NODETYPE" == "NTEST" ]; then TEST_ZOOKEEPER_ID=0; fi + +NODESTATE="$2" # default is inst, but we may have a spare type + +CFG=../config +HCFG_TEMPL=${CFG}/hadoop-cluster-config-template + +TEMPL_CDAP_CONF=../config/cdap-config-template +CDAP_CONF_FOLDER=../config/cdap-config + +HADOOP_CONF_FOLDER=../config/hadoop-cluster-config +HADOOP_SCRIPT_FOLDER=../pkgs/services + +IS_INITIAL=1 # Fresh cluster +MAXHOSTS=3 + +if [[ -n "$NODESTATE" && "inst" != "$NODESTATE" ]]; then + IS_ISINITIAL= +fi + +process_generic_node + +# [DE248724] configure tmpfiles.d +cp ../pkgs/ubuntu-files/cdap-hadoop-run.conf /usr/lib/tmpfiles.d/ +chmod 644 /usr/lib/tmpfiles.d/cdap-hadoop-run.conf + +cp ../pkgs/ubuntu-files/create_pid_dirs.sh "${__SERVICE_CONFIG_FOLDER__}"/ +chmod 755 "${__SERVICE_CONFIG_FOLDER__}"/create_pid_dirs.sh + +cp $HADOOP_SCRIPT_FOLDER/boot-time-cdap-vm-${NODETYPE}.sh "${__SERVICE_CONFIG_FOLDER__}"/boot-time-cdap-vm.sh +cp $HADOOP_SCRIPT_FOLDER/cdap-vm-services /etc/init.d +chmod +x /etc/init.d/cdap-vm-services + +# Ubuntu service enable: +# +update-rc.d cdap-vm-services defaults + +\. ./utils/cdap-nodetype-${NODETYPE}.sh diff --git a/cdap3vm/install-steps/install-cdap-pkgs.sh b/cdap3vm/install-steps/install-cdap-pkgs.sh new file mode 100755 index 0000000..c4c7ffa --- /dev/null +++ b/cdap3vm/install-steps/install-cdap-pkgs.sh @@ -0,0 +1,49 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +export JAVA_HOME=/opt/app/java/jdk/jdk170 +PATH=$JAVA_HOME/bin:$PATH + +## Assumption: Hadoop is already installed and configured +# +# hdfs.namespace is /cdap and the default property hdfs.user is yarn +# +sudo su hdfs -c "JAVA_HOME=/opt/app/java/jdk/jdk170 hadoop fs -mkdir -p /cdap && hadoop fs -chown yarn /cdap" + +# create a tx.snapshot subdirectory +# +sudo su hdfs -c "JAVA_HOME=/opt/app/java/jdk/jdk170 hadoop fs -mkdir -p /cdap/tx.snapshot && hadoop fs -chown yarn /cdap/tx.snapshot" + +## Configure apt-get repo for CDAP 3.5.x: +# +echo "Configuring CDAP 3.5 apt-get repo..." +sudo wget -O /etc/apt/sources.list.d/cask.list http://repository.cask.co/ubuntu/precise/amd64/cdap/3.5/cask.list + +## Add the Cask Public GPG Key: +# +echo "Adding the Cask public GPG key to apt..." +wget -O - http://repository.cask.co/ubuntu/precise/amd64/cdap/3.5/pubkey.gpg | sudo apt-key add - + +## Update apt cache: +# +echo "apt-get update..." +sudo apt-get -y update + +## Install CDAP packages: +echo "installing cdap packages..." +sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get -y install cdap-gateway cdap-kafka cdap-master cdap-security cdap-ui cdap-cli diff --git a/cdap3vm/install-steps/install-hortonworks-hadoop.sh b/cdap3vm/install-steps/install-hortonworks-hadoop.sh new file mode 100755 index 0000000..9e73eeb --- /dev/null +++ b/cdap3vm/install-steps/install-hortonworks-hadoop.sh @@ -0,0 +1,61 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +export JAVA_HOME=/opt/app/java/jdk/jdk170 + +## Configure apt-get repo for HortonWorks: +# +echo "Configuring HortonWorks apt-get repo..." +sudo wget http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.4.3.0/hdp.list -O /etc/apt/sources.list.d/hdp.list + +## Add the BigTop Public GPG Key: +# +# echo "Adding the HortonWorks public GPG key to apt..." +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 07513CAD + +## Update apt cache: +# +echo "apt-get update..." +sudo apt-get -y update + +## List Hadoop packages: +# +sudo apt-cache search hadoop +sudo apt-cache search hbase +sudo apt-cache search zookeeper + +## Install HortonWorks Hadoop packages: +# +# sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get install hadoop\* + +# sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get --print-uris install \ + +sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get -y install \ + hadoop hadoop-client hadoop-hdfs hadoop-hdfs-datanode hadoop-hdfs-namenode \ + hadoop-hdfs-zkfc hadoop-mapreduce hadoop-mapreduce-historyserver \ + hadoop-yarn hadoop-yarn-nodemanager \ + hbase hbase-master hbase-regionserver \ + zookeeper libhdfs0 + +## Fix file permissions for domain sockets +# +sudo chown hdfs:hadoop /var/lib/hadoop-hdfs +sudo chown mapred:hadoop /var/lib/hadoop-mapreduce +sudo chown yarn:hadoop /var/lib/hadoop-yarn +sudo chmod 775 /var/lib/hadoop-mapreduce + diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh new file mode 100644 index 0000000..d55f6ae --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/node-manager.sh + +wait_for_hbase_shell_OK +bash $MAINDIR/hbase-regionserver.sh + diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh new file mode 100644 index 0000000..d8affb9 --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/resource-manager.sh +bash $MAINDIR/node-manager.sh +bash $MAINDIR/job-history-server.sh + +wait_for_namenode +bash $MAINDIR/hbase-master.sh +bash $MAINDIR/hbase-regionserver.sh diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh new file mode 100644 index 0000000..eb9c9f3 --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/node-manager.sh + +wait_for_hbase_shell_OK +bash $MAINDIR/hbase-regionserver.sh +bash $MAINDIR/cdap.sh diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh new file mode 100644 index 0000000..f6140a4 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_namenode # will do datanode +inst_yarn_nodemanager + +wait_for_hbase_shell_OK +inst_hbase_regionserver + +# inst_cdap # inst_cdap will start CDAP services diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh new file mode 100644 index 0000000..4e87124 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_datanode +inst_yarn_resourcemanager +inst_yarn_nodemanager +inst_job_history_server + +wait_for_namenode +sleep 20 # for namenode to create required folders; also hbase master needs namenode + +inst_hbase_master +inst_hbase_regionserver + +# inst_cdap # inst_cdap will start CDAP services diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh new file mode 100644 index 0000000..52ef5fc --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_datanode +inst_yarn_nodemanager + +wait_for_hbase_shell_OK +inst_hbase_regionserver + +inst_cdap diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh b/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh new file mode 100644 index 0000000..0ae4b3d --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_namenode # will do datanode +inst_yarn_node_manager + +inst_cdap +inst_yarn_resource_manager +inst_job_history_server +inst_hbase_master +inst_hbase_regionserver diff --git a/cdap3vm/install-steps/utils/cdap-vm-services b/cdap3vm/install-steps/utils/cdap-vm-services new file mode 100644 index 0000000..281cd36 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-vm-services @@ -0,0 +1,42 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# description: Starts and stops the master service +# +### BEGIN INIT INFO +# Provides: cdap-vm-services +# Short-Description: Hadoop and CDAP services for this CDAP node +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +MAINDIR=/etc/hadoop/service-scripts/ + +if [ "start" = "$1" ]; then + bash $MAINDIR/boot-time-cdap-vm.sh + ( cd /opt/app/dcae-controller-service-cdap-cluster-manager ; sudo su dcae -c "JAVA_HOME=/opt/app/java/jdk/jdk170 bin/manager.sh start" ) +fi + + + diff --git a/cdap3vm/install-steps/utils/folder-creation-utils.sh b/cdap3vm/install-steps/utils/folder-creation-utils.sh new file mode 100644 index 0000000..8a8f82a --- /dev/null +++ b/cdap3vm/install-steps/utils/folder-creation-utils.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +copy_hadoop_conf_files() { + srcfolder=${HADOOP_CONF_FOLDER}/"$1" + destfolder="$2" + cp -a $srcfolder/* $destfolder/ +} + +copy_cdap_conf_files() { + srcfolder=${CDAP_CONF_FOLDER} + destfolder="${__CDAP_CONF_DIR__}" + cp -a $srcfolder/cdap-* $srcfolder/log* $destfolder/ + # Hack for a common.sh file -- TODO: Make it more general + cp -a $srcfolder/common/common.sh ${__CDAP_INST_FOLDER__}/ui/bin/common.sh +} + + +setup_hadoop_service_scripts() { + for x in "$@"; do + srcscript=${HADOOP_SCRIPT_FOLDER}/"${x}" + destscript="${__SERVICE_CONFIG_FOLDER__}/${x}" + cp -a $srcscript $destscript + done +} + +run_service_setup_script() { + script="$1" + user="$2" + runner=${HADOOP_SERVICE_SETUP_FOLDER}/service-setup.sh + srcscript=${HADOOP_SERVICE_SETUP_FOLDER}/"${script}" + bash ${runner} ${script} ${user} # should not give full path +} + +create_folder() { + # Create a folder with a group and mode + name=$1 # if there are spaces, there will be multiple folders + user="$2" + group="$3" + mode="$4" + + mkdir -p $name # if there are spaces, there will be multiple folders + chown -R "$user":"$group" $name + chmod -R "$mode" $name +} + +inst_namenode() { + if [ -n "$IS_INITIAL" ]; then + rm -rf ${__DFS_NAME_DIR__} # TODO: Move away from force delete? + create_folder ${__DFS_NAME_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + run_service_setup_script namenode-setup.sh ${__HDFS_USER__} + setup_hadoop_service_scripts namenode.sh + else + exit -1; # We do not know what to do + fi +} + +inst_secondary_namenode() { + if [ -n "$IS_INITIAL" ]; then + create_folder ${__FS_CHECKPOINT_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + setup_hadoop_service_scripts secondary-namenode.sh + else + exit -1; # We do not know what to do + fi +} + +inst_yarn_basic() { + create_folder ${__YARN_LOCAL_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + create_folder ${__YARN_LOCAL_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 +} + +inst_datanode_basic() { + create_folder ${__HDFS_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + rm -rf ${__DFS_DATA_DIR__} # TODO: Move away from force delete? + create_folder ${__DFS_DATA_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 750 + setup_hadoop_service_scripts datanode.sh +} + +inst_datanode() { + run_service_setup_script datanode-start.sh ${__HDFS_USER__} +} + +inst_hbase_basic() { + create_folder ${__HBASE_LOG_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750 + create_folder ${__HBASE_PID_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750 + rm -rf ${__HBASE_CONF_DIR__} + create_folder ${__HBASE_CONF_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 755 + copy_hadoop_conf_files hbase ${__HBASE_CONF_DIR__} +} + +inst_hbase_master() { + run_service_setup_script hbase-master-start.sh ${__HBASE_USER__} + setup_hadoop_service_scripts hbase-master.sh +} + +inst_hbase_regionserver() { + run_service_setup_script hbase-regionserver-start.sh ${__HBASE_USER__} + setup_hadoop_service_scripts hbase-regionserver.sh +} + +inst_core_files() { + # For all nodes + # YARN logs + create_folder ${__YARN_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # HDFS PID folder + create_folder ${__HDFS_PID_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # YARN Node Manager Recovery Directory + create_folder ${__YARN_NODEMANAGER_RECOVERY_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # YARN PID folder + create_folder ${__YARN_PID_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # JobHistory server logs + create_folder ${__MAPRED_LOG_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755 + # JobHistory PID folder + create_folder ${__MAPRED_PID_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755 + # hadoop conf dir + rm -rf ${__HADOOP_CONF_DIR__} + create_folder ${__HADOOP_CONF_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # dfs.exclude -- before things are created + touch ${__HADOOP_CONF_DIR__}/dfs.exclude + copy_hadoop_conf_files core_hadoop ${__HADOOP_CONF_DIR__} +} + +inst_yarn_nodemanager() { + run_service_setup_script node-manager-start.sh ${__YARN_USER__} + setup_hadoop_service_scripts node-manager.sh +} + +inst_yarn_resourcemanager() { + run_service_setup_script resource-manager-start.sh ${__YARN_USER__} + setup_hadoop_service_scripts resource-manager.sh +} + +inst_job_history_server() { + run_service_setup_script job-history-setup-01-as-root.sh # no need for username then + run_service_setup_script job-history-setup-02-as-hdfs.sh ${__HDFS_USER__} + setup_hadoop_service_scripts job-history-server.sh +} + +inst_cdap() { + # note: cluster-prep is done along with namenode + run_service_setup_script cdap-setup.sh # as root + setup_hadoop_service_scripts cdap.sh + copy_cdap_conf_files + run_service_setup_script cdap-start.sh # as root +} + +inst_zookeeper() { + # Zookeeper data folder + create_folder ${__ZOOKEEPER_DATA_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + # Zookeeper logs + create_folder ${__ZOOKEEPER_LOG_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + create_folder ${__ZKFC_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # Zookeeper PID folder + create_folder ${__ZOOKEEPER_PID_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + # Put the Zookeeper ID into its folder + get_zookeeper_id > ${__ZOOKEEPER_DATA_DIR__}/myid + # Clean up Zookeeper conf folder + rm -rf ${__ZOOKEEPER_CONF_DIR__} + create_folder ${__ZOOKEEPER_CONF_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + chmod 755 ${__ZOOKEEPER_CONF_DIR__}/.. # Parent folder of zookeeper + # Copy zookeeper files + copy_hadoop_conf_files zookeeper ${__ZOOKEEPER_CONF_DIR__} + run_service_setup_script zookeeper-start.sh ${__ZOOKEEPER_USER__} + run_service_setup_script zookeeper-zkfc-start.sh ${__HDFS_USER__} +} + +get_cdap_host_id() { + # we will set this based on the hostname + base=$(hostname -s) + base=$(echo $base | sed 's/^.*[^0-9]\([0-9][0-9]*\)/\1/g') + base=${base: -1} # get last digit. TODO: Need to make it scale? + echo $base +} + +get_zookeeper_id() { + echo $(get_cdap_host_id) +} + +get_zookeeper_id_single_digit() { + base=$(get_cdap_host_id) + echo ${base: -1} +} + +process_generic_node() { + + create_folder ${__SERVICE_CONFIG_FOLDER__} root root 755 + + inst_core_files # config files, creating, copying etc. + setup_hadoop_service_scripts utility-scripts.sh + setup_hadoop_service_scripts service-start.sh + setup_hadoop_service_scripts zookeeper.sh + setup_hadoop_service_scripts zookeeper-zkfc.sh + + inst_zookeeper + inst_datanode_basic + inst_hbase_basic + inst_yarn_basic +} + +wait_for_remote_service() { + remote_host="$1" + remote_port="$2" + sleep_time=${3:-30} # default of 30 seconds between retries + + # keep checking remote_host's remote_port every sleep_time seconds till we get a connection + while ( ! nc $remote_host $remote_port < /dev/null ); do sleep $sleep_time ; done +} + +wait_for_namenode() { + # keep checking namenode's port 8020 till it is up -- do it every 30 seconds + wait_for_remote_service ${__HDP_NAMENODE__} 8020 30 +} + +wait_for_hbase_master_start() { + # keep checking hbase master's port 16000 till it is up -- do it every 30 seconds + wait_for_remote_service ${__HDP_HBASE_MASTER__} 16000 30 +} + +wait_for_hbase_shell_OK() { + # run hbase shell and see if we connect to hbase... Better than waiting for ports + while ( echo list | hbase shell 2>&1 | grep ^ERROR > /dev/null ); do + sleep 30 + done +} diff --git a/cdap3vm/install-steps/utils/generate-hosts.sh b/cdap3vm/install-steps/utils/generate-hosts.sh new file mode 100644 index 0000000..b080a1e --- /dev/null +++ b/cdap3vm/install-steps/utils/generate-hosts.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAXHOSTID=2 + +fullhostname=$(hostname -f) +basehostname=${fullhostname%%.*} +restname=${fullhostname#*.} + +basenum=$(echo $base | sed 's/^.*[^0-9]\([0-9][0-9]*\)/\1/g') +basenum=${basenum##0*} # remove leading zeros + +if [[ "${basenum}" -ge 0 && "${basenum}" -le ${MAXHOSTS} ]]; then + for x in $(seq 0 $MAXHOSTID); do + echo "__DCAE_CDAP_NODE${x}__=${basehostname::-1}${x}" + # echo "__DCAE_CDAP_NODE${x}__=${basehostname::-1}${x}.${restname}" + # echo "_CLUSTER_SERVER_PREFIX__0${x}=${basehostname::-1}${x}.${restname}" + done +fi + + diff --git a/cdap3vm/install-steps/utils/replace-kv.py b/cdap3vm/install-steps/utils/replace-kv.py new file mode 100644 index 0000000..acd9782 --- /dev/null +++ b/cdap3vm/install-steps/utils/replace-kv.py @@ -0,0 +1,63 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +from __future__ import print_function +import os +import re +import sys + +""" +Replace patterns in a file with the corresponding replacements. + +Args: OriginalFile OutFile XRefFile +""" + + +def process_kv_file(fname): + """ + Read a simple properties file, X=Y type; file may contain comment lines and blanks. Return a dict + Leading comments are '^\s*#', while blank lines aare '^\s*$' + Key cannot contain '=', but value can + """ + ignore_pattern = re.compile(r'^\s*(#|$)') + with open(fname) as fid: + all_lines = [ line.strip() for line in fid if not re.match(ignore_pattern, line) ] + return dict( line.partition('=')[::2] for line in all_lines ) + + +def replace_kv(fname, outfname, *xref_files): + """ + Read a file, and perform multiple search replace using key-values in xref. + Keys have to be simple keys + """ + xref = {} + for xref_file in xref_files: + xref.update(process_kv_file(xref_file)) + pattern = re.compile("|".join(xref.keys())) + with open(outfname, 'w') as outf: + with open(fname) as fid: + all_text = fid.read() + outf.write(pattern.sub(lambda m: xref[m.group(0)], all_text)) + + +if __name__ == "__main__": + if len(sys.argv) >= 3: + replace_kv(*sys.argv[1:]) + else: + print("Usage: {} <{}> <{}> <{}> [{}]".format( + sys.argv[0], "InputFile", "OutPutFile", "XREF-DictFile[s]", "..."), file=sys.stderr); diff --git a/cdap3vm/install.sh b/cdap3vm/install.sh new file mode 100644 index 0000000..d2947ac --- /dev/null +++ b/cdap3vm/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Default will be a production installation +# Any test installation requires passing arguments to different steps + +cd $(dirname $0)/install-steps + +bash 01-generate-host-ids-configs.sh +bash 02-user-creation.sh +#bash 03-hadoop-rpms.sh +bash install-hortonworks-hadoop.sh +bash install-cdap-pkgs.sh +bash 04-folder-creation.sh + diff --git a/cdap3vm/pkgs/services-setup/cdap-setup.sh b/cdap3vm/pkgs/services-setup/cdap-setup.sh new file mode 100644 index 0000000..137e9e3 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/cdap-setup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# CDAP setup on all machines! + +echo "Should have already installed CDAP RPMs" + +exit + +RPM_DIR="../installation-steps" + +( + cd $RPM_DIR/04-cdap-pkgs + rpm -Uvh * +) + + diff --git a/cdap3vm/pkgs/services-setup/datanode-start.sh b/cdap3vm/pkgs/services-setup/datanode-start.sh new file mode 100644 index 0000000..77a0f4e --- /dev/null +++ b/cdap3vm/pkgs/services-setup/datanode-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + diff --git a/cdap3vm/pkgs/services-setup/hbase-master-start.sh b/cdap3vm/pkgs/services-setup/hbase-master-start.sh new file mode 100644 index 0000000..3e0db6b --- /dev/null +++ b/cdap3vm/pkgs/services-setup/hbase-master-start.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh b/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh new file mode 100644 index 0000000..259091f --- /dev/null +++ b/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +sleep 100 # Wait for all things to sync up? +/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh b/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh new file mode 100644 index 0000000..9c5e992 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +chown -R root:hadoop /usr/hdp/current/hadoop-yarn*/bin/container-executor +chmod -R 6050 /usr/hdp/current/hadoop-yarn*/bin/container-executor diff --git a/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh b/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh new file mode 100644 index 0000000..4b365f1 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +hdfs dfs -mkdir -p /mr-history/tmp +hdfs dfs -chmod -R 1777 /mr-history/tmp + +hdfs dfs -mkdir -p /mr-history/done +hdfs dfs -chmod -R 1777 /mr-history/done +hdfs dfs -chown -R mapred:hdfs /mr-history + +hdfs dfs -mkdir -p /app-logs +hdfs dfs -chmod -R 1777 /app-logs + +hdfs dfs -chown yarn:hdfs /app-logs + + + diff --git a/cdap3vm/pkgs/services-setup/namenode-setup.sh b/cdap3vm/pkgs/services-setup/namenode-setup.sh new file mode 100644 index 0000000..54c9d1c --- /dev/null +++ b/cdap3vm/pkgs/services-setup/namenode-setup.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/bin/hdfs namenode -format -force +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + +sleep 30 # Wait for services to come up? + +hdfs dfs -mkdir -p /hdp/apps/2.3.2.0-2950/mapreduce/ +hdfs dfs -put /usr/hdp/current/hadoop-client/mapreduce.tar.gz /hdp/apps/2.3.2.0-2950/mapreduce/ +hdfs dfs -chown -R hdfs:hadoop /hdp +hdfs dfs -chmod -R 555 /hdp/apps/2.3.2.0-2950/mapreduce +hdfs dfs -chmod 444 /hdp/apps/2.3.2.0-2950/mapreduce/mapreduce.tar.gz + +# We will try to set up general CDAP related stuff (cluster-prep) here +( +hdfs dfs -mkdir -p /user/yarn +hdfs dfs -chown yarn /user/yarn +hdfs dfs -chgrp yarn /user/yarn + +hdfs dfs -mkdir -p /cdap +hdfs dfs -chown yarn /cdap + +hdfs dfs -mkdir -p /cdap/tx.snapshot +hdfs dfs -chown yarn /cdap/tx.snapshot +) & diff --git a/cdap3vm/pkgs/services-setup/node-manager-start.sh b/cdap3vm/pkgs/services-setup/node-manager-start.sh new file mode 100644 index 0000000..d0403ee --- /dev/null +++ b/cdap3vm/pkgs/services-setup/node-manager-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start nodemanager + diff --git a/cdap3vm/pkgs/services-setup/resource-manager-start.sh b/cdap3vm/pkgs/services-setup/resource-manager-start.sh new file mode 100644 index 0000000..bf8366f --- /dev/null +++ b/cdap3vm/pkgs/services-setup/resource-manager-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start resourcemanager + diff --git a/cdap3vm/pkgs/services-setup/service-setup.sh b/cdap3vm/pkgs/services-setup/service-setup.sh new file mode 100644 index 0000000..e3733f0 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/service-setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +setup_script="${1}" + +if [ -z "$2" ]; then + bash ${MAINLOC}/${setup_script} +else + su "$2" -c "bash ${MAINLOC}/${setup_script}" +fi diff --git a/cdap3vm/pkgs/services-setup/zookeeper-start.sh b/cdap3vm/pkgs/services-setup/zookeeper-start.sh new file mode 100644 index 0000000..64c4b7d --- /dev/null +++ b/cdap3vm/pkgs/services-setup/zookeeper-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="start" + +export ZOOCFGDIR=/etc/zookeeper/conf +export ZOOCFG=zoo.cfg +source ${ZOOCFGDIR}/zookeeper-env.sh +/usr/hdp/current/zookeeper-server/bin/zkServer.sh "$ACTION" + + diff --git a/cdap3vm/pkgs/services/datanode.sh b/cdap3vm/pkgs/services/datanode.sh new file mode 100644 index 0000000..77a0f4e --- /dev/null +++ b/cdap3vm/pkgs/services/datanode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + diff --git a/cdap3vm/pkgs/services/hbase-master.sh b/cdap3vm/pkgs/services/hbase-master.sh new file mode 100644 index 0000000..3e0db6b --- /dev/null +++ b/cdap3vm/pkgs/services/hbase-master.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/pkgs/services/hbase-regionserver.sh b/cdap3vm/pkgs/services/hbase-regionserver.sh new file mode 100644 index 0000000..259091f --- /dev/null +++ b/cdap3vm/pkgs/services/hbase-regionserver.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +sleep 100 # Wait for all things to sync up? +/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/pkgs/services/job-history-server.sh b/cdap3vm/pkgs/services/job-history-server.sh new file mode 100644 index 0000000..de1f30a --- /dev/null +++ b/cdap3vm/pkgs/services/job-history-server.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-mapreduce-historyserver/sbin/mr-jobhistory-daemon.sh --config /etc/hadoop/conf start historyserver + diff --git a/cdap3vm/pkgs/services/namenode.sh b/cdap3vm/pkgs/services/namenode.sh new file mode 100644 index 0000000..2814f28 --- /dev/null +++ b/cdap3vm/pkgs/services/namenode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode + diff --git a/cdap3vm/pkgs/services/node-manager.sh b/cdap3vm/pkgs/services/node-manager.sh new file mode 100644 index 0000000..d0403ee --- /dev/null +++ b/cdap3vm/pkgs/services/node-manager.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start nodemanager + diff --git a/cdap3vm/pkgs/services/resource-manager.sh b/cdap3vm/pkgs/services/resource-manager.sh new file mode 100644 index 0000000..bf8366f --- /dev/null +++ b/cdap3vm/pkgs/services/resource-manager.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start resourcemanager + diff --git a/cdap3vm/pkgs/services/secondary-namenode.sh b/cdap3vm/pkgs/services/secondary-namenode.sh new file mode 100644 index 0000000..2c6dc0f --- /dev/null +++ b/cdap3vm/pkgs/services/secondary-namenode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start secondarynamenode + diff --git a/cdap3vm/pkgs/services/service-start.sh b/cdap3vm/pkgs/services/service-start.sh new file mode 100644 index 0000000..a3094f9 --- /dev/null +++ b/cdap3vm/pkgs/services/service-start.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +service_name="${1%.sh}" + +username=${2:-$service_name} + +# Usage: $0 zookeeper will result in su zookeeper -c +su "$username" -c "bash ${MAINLOC}/${service_name}.sh start" diff --git a/cdap3vm/pkgs/services/zookeeper.sh b/cdap3vm/pkgs/services/zookeeper.sh new file mode 100644 index 0000000..9f86d9d --- /dev/null +++ b/cdap3vm/pkgs/services/zookeeper.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="$1" + +case "$ACTION" in + start|stop|restart|status ) + export ZOOCFGDIR=/etc/zookeeper/conf + export ZOOCFG=zoo.cfg + source ${ZOOCFGDIR}/zookeeper-env.sh + /usr/hdp/current/zookeeper-server/bin/zkServer.sh "$ACTION" ;; + * ) + echo "Usage: $0 <start|stop|restart|status>" + exit -1 ;; +esac + diff --git a/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf b/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf new file mode 100644 index 0000000..770689c --- /dev/null +++ b/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf @@ -0,0 +1,34 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# this file is part of dcae-cdap-small-hadoop +# it belongs in /usr/lib/tmpfiles.d +# +# [DE248724] Hadoop /var/run PID directories not persisted between +# reboots on CentOS7 +# +# +d /var/run/cdap 0755 cdap cdap - +d /var/run/hbase 0750 hbase hadoop - +d /var/run/zookeeper 0755 zookeeper hadoop - +d /var/run/hadoop 0755 root root - +d /var/run/hadoop/hdfs 0755 hdfs hadoop - +d /var/run/hadoop/yarn 0755 yarn hadoop - +d /var/run/hadoop/yarn/yarn 0755 yarn yarn - +d /var/run/hadoop/mapred 0755 mapred hadoop - +d /var/run/hadoop/mapreduce 0775 mapred hadoop - diff --git a/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh b/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh new file mode 100644 index 0000000..2812b31 --- /dev/null +++ b/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh @@ -0,0 +1,48 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# [DE248724] Hadoop /var/run PID directories not persisted between +# reboots +# Ubuntu 14 does not actually use /usr/lib/tmpfs.d, so need to +# handle creating PID dirs in init scripts. + +# zookeeper: +sudo mkdir -p -m0755 /var/run/zookeeper +sudo chown zookeeper:hadoop /var/run/zookeeper + +# hbase: +sudo mkdir -p -m0750 /var/run/hbase +sudo chown hbase:hadoop /var/run/hbase + +# hdfs: +sudo mkdir -p -m0755 /var/run/hadoop/hdfs +sudo chown hdfs:hadoop /var/run/hadoop/hdfs + +# yarn: +sudo mkdir -p -m0755 /var/run/hadoop/yarn +sudo chown yarn:hadoop /var/run/hadoop/yarn +sudo mkdir -p -m0755 /var/run/hadoop/yarn/yarn +sudo chown yarn:hadoop /var/run/hadoop/yarn/yarn + +# mapred: +sudo mkdir -p -m0755 /var/run/hadoop/mapred +sudo chown mapred:hadoop /var/run/hadoop/mapred + +# mapreduce: +sudo mkdir -p -m0755 /var/run/hadoop/mapreduce +sudo chown mapred:hadoop /var/run/hadoop/mapreduce |