diff options
Diffstat (limited to 'cdap3vm/config/hadoop-cluster-config-template/hbase')
9 files changed, 997 insertions, 0 deletions
diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml new file mode 100644 index 0000000..b021418 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml @@ -0,0 +1,147 @@ +<!--Thu Jun 18 04:06:19 2015--> + <configuration> + + <property> + <name>fs.defaultFS</name> + <value>hdfs://__HDP_NAMENODE__:8020</value> + <final>true</final> + </property> + + <property> + <name>fs.trash.interval</name> + <value>360</value> + </property> + + <property> + <name>ha.failover-controller.active-standby-elector.zk.op.retries</name> + <value>120</value> + </property> + + <property> + <name>hadoop.http.authentication.simple.anonymous.allowed</name> + <value>true</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.falcon.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hcat.hosts</name> + <value>__HDP_WEBHCAT_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hdfs.hosts</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.groups</name> + <value>users</value> + </property> + + <property> + <name>hadoop.proxyuser.hive.hosts</name> + <value>__HDP_HIVE_SERVER__</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.groups</name> + <value>*</value> + </property> + + <property> + <name>hadoop.proxyuser.oozie.hosts</name> + <value>__HDP_OOZIE_SERVER__</value> + </property> + + <property> + <name>hadoop.security.auth_to_local</name> + <value>DEFAULT</value> + </property> + + <property> + <name>hadoop.security.authentication</name> + <value>simple</value> + </property> + + <property> + <name>hadoop.security.authorization</name> + <value>false</value> + </property> + + <property> + <name>hadoop.security.key.provider.path</name> + <value></value> + </property> + + <property> + <name>io.compression.codecs</name> + <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec</value> + </property> + + <property> + <name>io.file.buffer.size</name> + <value>131072</value> + </property> + + <property> + <name>io.serializations</name> + <value>org.apache.hadoop.io.serializer.WritableSerialization</value> + </property> + + <property> + <name>ipc.client.connect.max.retries</name> + <value>50</value> + </property> + + <property> + <name>ipc.client.connection.maxidletime</name> + <value>30000</value> + </property> + + <property> + <name>ipc.client.idlethreshold</name> + <value>8000</value> + </property> + + <property> + <name>ipc.server.tcpnodelay</name> + <value>true</value> + </property> + + <property> + <name>mapreduce.jobtracker.webinterface.trusted</name> + <value>false</value> + </property> + +<!-- + <property> + <name>net.topology.script.file.name</name> + <value>/etc/hadoop/conf/topology_script.py</value> + </property> +--> + + <property> + <name>proxyuser_group</name> + <value>users</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..0911e46 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties @@ -0,0 +1,50 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + + +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.slave.host.name=rohit-companionfiles-1.c.pramod-thangali.internal +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd new file mode 100644 index 0000000..9fb87d7 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd @@ -0,0 +1,87 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Set environment variables here. + +@rem The java implementation to use. Java 1.7+ required. +@rem set JAVA_HOME=c:\apps\java + +@rem Extra Java CLASSPATH elements. Optional. +@rem set HBASE_CLASSPATH= + +@rem The maximum amount of heap to use. Default is left to JVM default. +@rem set HBASE_HEAPSIZE=1000 + +@rem Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +@rem offheap, set the value to "8G". +@rem set HBASE_OFFHEAPSIZE=1000 + +@rem For example, to allocate 8G of offheap, to 8G: +@rem etHBASE_OFFHEAPSIZE=8G + +@rem Extra Java runtime options. +@rem Below are what we set by default. May only work with SUN JVM. +@rem For more on why as well as other possible settings, +@rem see http://wiki.apache.org/hadoop/PerformanceTuning +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +@rem @rem See TestIPv6NIOServerSocketChannel. +set HBASE_OPTS="-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stack=true" + +@rem Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ +set HBASE_MASTER_OPTS=%HBASE_MASTER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" +set HBASE_REGIONSERVER_OPTS=%HBASE_REGIONSERVER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" + +@rem Uncomment below to enable java garbage collection logging for the server-side processes +@rem this enables basic gc logging for the server processes to the .out file +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" "-XX:+UseGCLogFileRotation" "-XX:NumberOfGCLogFiles=1" "-XX:GCLogFileSize=512M" %HBASE_GC_OPTS% + +@rem Uncomment below to enable java garbage collection logging for the client processes in the .out file. +@rem set CLIENT_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +@rem set HBASE_USE_GC_LOGFILE=true + +@rem Uncomment and adjust to enable JMX exporting +@rem See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +@rem More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +@rem +@rem set HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false" "-Dcom.sun.management.jmxremote.authenticate=false" +@rem set HBASE_MASTER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10101" +@rem set HBASE_REGIONSERVER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10102" +@rem set HBASE_THRIFT_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10103" +@rem set HBASE_ZOOKEEPER_OPTS=%HBASE_JMX_BASE% -Dcom.sun.management.jmxremote.port=10104" + +@rem File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +@rem set HBASE_REGIONSERVERS=%HBASE_HOME%\conf\regionservers + +@rem Where log files are stored. $HBASE_HOME/logs by default. +@rem set HBASE_LOG_DIR=%HBASE_HOME%\logs + +@rem A string representing this instance of hbase. $USER by default. +@rem set HBASE_IDENT_STRING=%USERNAME% + +@rem Seconds to sleep between slave commands. Unset by default. This +@rem can be useful in large clusters, where, e.g., slave rsyncs can +@rem otherwise arrive faster than the master can service them. +@rem set HBASE_SLAVE_SLEEP=0.1 + +@rem Tell HBase whether it should manage it's own instance of Zookeeper or not. +@rem set HBASE_MANAGES_ZK=true diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh new file mode 100644 index 0000000..ce60e00 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh @@ -0,0 +1,65 @@ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME=__JAVA_HOME__ + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-__HBASE_CONF_DIR__} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:__HBASE_LOG_DIR__/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR=__HBASE_LOG_DIR__ + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR=/var/run/hbase + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + + + + +export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile=__HBASE_LOG_DIR__/hs_err_pid%p.log" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx4096m $JDK_DEPENDED_OPTS" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn512m -XX:CMSInitiatingOccupancyFraction=70 -Xms1024m -Xmx8192m $JDK_DEPENDED_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml new file mode 100644 index 0000000..0392423 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml @@ -0,0 +1,19 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + </property> + + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + </property> + + </configuration>
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml new file mode 100644 index 0000000..4c75e95 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml @@ -0,0 +1,234 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + </property> + + <property> + <name>hbase.bulkload.staging.dir</name> + <value>/apps/hbase/staging</value> + </property> + + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>1048576</value> + </property> + + <property> + <name>hbase.client.retries.number</name> + <value>35</value> + </property> + + <property> + <name>hbase.client.scanner.caching</name> + <value>100</value> + </property> + + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + + <property> + <name>hbase.coprocessor.master.classes</name> + <value></value> + </property> + + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value> + </property> + + <property> + <name>hbase.defaults.for.version.skip</name> + <value>true</value> + </property> + + <property> + <name>hbase.hregion.majorcompaction</name> + <value>604800000</value> + </property> + + <property> + <name>hbase.hregion.majorcompaction.jitter</name> + <value>0.50</value> + </property> + + <property> + <name>hbase.hregion.max.filesize</name> + <value>10737418240</value> + </property> + + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>4</value> + </property> + + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>134217728</value> + </property> + + <property> + <name>hbase.hregion.memstore.mslab.enabled</name> + <value>true</value> + </property> + + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>10</value> + </property> + + <property> + <name>hbase.hstore.compaction.max</name> + <value>10</value> + </property> + + <property> + <name>hbase.hstore.compactionThreshold</name> + <value>3</value> + </property> + + <property> + <name>hbase.local.dir</name> + <value>${hbase.tmp.dir}/local</value> + </property> + + <property> + <name>hbase.master.info.bindAddress</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>hbase.master.info.port</name> + <value>16010</value> + </property> + + <property> + <name>hbase.master.port</name> + <value>16000</value> + </property> + + <property> + <name>hbase.region.server.rpc.scheduler.factory.class</name> + <value></value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.lowerLimit</name> + <value>0.38</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.size</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.upperLimit</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.handler.count</name> + <value>30</value> + </property> + + <property> + <name>hbase.regionserver.info.port</name> + <value>16030</value> + </property> + + <property> + <name>hbase.regionserver.port</name> + <value>16020</value> + </property> + + <property> + <name>hbase.regionserver.wal.codec</name> + <value>org.apache.hadoop.hbase.regionserver.wal.WALCellCodec</value> + </property> + + <property> + <name>hbase.rootdir</name> + <value>hdfs://__HDP_NAMENODE__:8020/apps/hbase/data</value> + </property> + + <property> + <name>hbase.rpc.controllerfactory.class</name> + <value></value> + </property> + + <property> + <name>hbase.rpc.protection</name> + <value>authentication</value> + </property> + + <property> + <name>hbase.rpc.timeout</name> + <value>90000</value> + </property> + + <property> + <name>hbase.security.authentication</name> + <value>simple</value> + </property> + + <property> + <name>hbase.security.authorization</name> + <value>false</value> + </property> + + <property> + <name>hbase.superuser</name> + <value>hbase</value> + </property> + + <property> + <name>hbase.tmp.dir</name> + <value>/tmp/hbase-${user.name}</value> + </property> + + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value>__HDP_ZOOKEEPER_HOSTS__</value> + </property> + + <property> + <name>hbase.zookeeper.useMulti</name> + <value>true</value> + </property> + + <property> + <name>hfile.block.cache.size</name> + <value>0.40</value> + </property> + + <property> + <name>phoenix.functions.allowUserDefinedFunctions</name> + <value> </value> + </property> + + <property> + <name>phoenix.query.timeoutMs</name> + <value>60000</value> + </property> + + <property> + <name>zookeeper.session.timeout</name> + <value>90000</value> + </property> + + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase-unsecure</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml new file mode 100644 index 0000000..6367375 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml @@ -0,0 +1,281 @@ +<!--Thu Jun 18 04:06:21 2015--> + <configuration> + + <property> + <name>dfs.block.access.token.enable</name> + <value>true</value> + </property> + + <property> + <name>dfs.blockreport.initialDelay</name> + <value>120</value> + </property> + + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit</name> + <value>true</value> + </property> + + <property> + <name>dfs.client.read.shortcircuit.streams.cache.size</name> + <value>4096</value> + </property> + + <property> + <name>dfs.client.retry.policy.enabled</name> + <value>false</value> + </property> + + <property> + <name>dfs.cluster.administrators</name> + <value> hdfs</value> + </property> + + <property> + <name>dfs.datanode.address</name> + <value>0.0.0.0:50010</value> + </property> + + <property> + <name>dfs.datanode.balance.bandwidthPerSec</name> + <value>6250000</value> + </property> + + <property> + <name>dfs.datanode.data.dir</name> + <value>/opt/data/hdfs/dn</value> + </property> + + <property> + <name>dfs.datanode.data.dir.perm</name> + <value>750</value> + </property> + + <property> + <name>dfs.datanode.du.reserved</name> + <value>1073741824</value> + </property> + + <property> + <name>dfs.datanode.failed.volumes.tolerated</name> + <value>0</value> + </property> + + <property> + <name>dfs.datanode.http.address</name> + <value>0.0.0.0:50075</value> + </property> + + <property> + <name>dfs.datanode.https.address</name> + <value>0.0.0.0:50475</value> + </property> + + <property> + <name>dfs.datanode.ipc.address</name> + <value>0.0.0.0:8010</value> + </property> + + <property> + <name>dfs.datanode.max.transfer.threads</name> + <value>16384</value> + </property> + + <property> + <name>dfs.domain.socket.path</name> + <value>/var/lib/hadoop-hdfs/dn_socket</value> + </property> + + <property> + <name>dfs.encryption.key.provider.uri</name> + <value></value> + </property> + + <property> + <name>dfs.heartbeat.interval</name> + <value>3</value> + </property> + + <property> + <name>dfs.hosts.exclude</name> + <value>/etc/hadoop/conf/dfs.exclude</value> + </property> + + <property> + <name>dfs.http.policy</name> + <value>HTTP_ONLY</value> + </property> + + <property> + <name>dfs.https.port</name> + <value>50470</value> + </property> + + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/hadoop/hdfs/journalnode</value> + </property> + + <property> + <name>dfs.journalnode.http-address</name> + <value>0.0.0.0:8480</value> + </property> + + <property> + <name>dfs.journalnode.https-address</name> + <value>0.0.0.0:8481</value> + </property> + + <property> + <name>dfs.namenode.accesstime.precision</name> + <value>0</value> + </property> + + <property> + <name>dfs.namenode.audit.log.async</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.read.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.avoid.write.stale.datanode</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.dir</name> + <value>__FS_CHECKPOINT_DIR__</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.edits.dir</name> + <value>${dfs.namenode.checkpoint.dir}</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.period</name> + <value>21600</value> + </property> + + <property> + <name>dfs.namenode.checkpoint.txns</name> + <value>1000000</value> + </property> + + <property> + <name>dfs.namenode.fslock.fair</name> + <value>false</value> + </property> + + <property> + <name>dfs.namenode.handler.count</name> + <value>200</value> + </property> + + <property> + <name>dfs.namenode.http-address</name> + <value>__HDP_NAMENODE__:50070</value> + <final>true</final> + </property> + + <property> + <name>dfs.namenode.https-address</name> + <value>__HDP_NAMENODE__:50470</value> + </property> + + <property> + <name>dfs.namenode.name.dir</name> + <value>__DFS_NAME_DIR__</value> + </property> + + <property> + <name>dfs.namenode.name.dir.restore</name> + <value>true</value> + </property> + + <property> + <name>dfs.namenode.rpc-address</name> + <value>__HDP_NAMENODE__:8020</value> + </property> + + <property> + <name>dfs.namenode.safemode.threshold-pct</name> + <value>1</value> + </property> + + <property> + <name>dfs.namenode.secondary.http-address</name> + <value>__HDP_SECONDARY_NAMENODE__:50090</value> + </property> + + <property> + <name>dfs.namenode.stale.datanode.interval</name> + <value>30000</value> + </property> + + <property> + <name>dfs.namenode.startup.delay.block.deletion.sec</name> + <value>3600</value> + </property> + + <property> + <name>dfs.namenode.write.stale.datanode.ratio</name> + <value>1.0f</value> + </property> + + <property> + <name>dfs.permissions.enabled</name> + <value>true</value> + </property> + + <property> + <name>dfs.permissions.superusergroup</name> + <value>hdfs</value> + </property> + + <property> + <name>dfs.replication</name> + <value>3</value> + </property> + + <property> + <name>dfs.replication.max</name> + <value>50</value> + </property> + + <property> + <name>dfs.support.append</name> + <value>true</value> + <final>true</final> + </property> + + <property> + <name>dfs.webhdfs.enabled</name> + <value>false</value> + </property> + + <property> + <name>fs.permissions.umask-mode</name> + <value>022</value> + </property> + + <property> + <name>nfs.exports.allowed.hosts</name> + <value>* rw</value> + </property> + + <property> + <name>nfs.file.dump.dir</name> + <value>/tmp/.hdfs-nfs</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties new file mode 100644 index 0000000..3737eac --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties @@ -0,0 +1,111 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=INFO +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers new file mode 100644 index 0000000..96dc196 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers @@ -0,0 +1,3 @@ +__DCAE_CDAP_NODE0__ +__DCAE_CDAP_NODE1__ +__DCAE_CDAP_NODE2__ |