diff options
| author |   lj1412 <lji@research.att.com> | 2017-02-14 15:10:55 +0000 |
|---|---|---|
| committer | lj1412 <lji@research.att.com> | 2017-02-14 15:10:57 +0000 |
| commit | 39fd75bdb79ac63a05dadda0ba32792e49783031 (patch) | |
| tree | d63bf1c88c1596031cdcb38c81c6fd12f17d831d /cdap3vm/config/hadoop-cluster-config-template/hive | |
| parent | 58c3d798750a223440a944661e5dec5e33c0c0e1 (diff) | |
Init dcae.apod.cdap
Change-Id: I0d1f1ba04f8ba7a2a276e4e382eb07ae9d6a7652
Signed-off-by: lj1412 <lji@research.att.com>
Diffstat (limited to 'cdap3vm/config/hadoop-cluster-config-template/hive')
9 files changed, 1335 insertions, 0 deletions
diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template new file mode 100644 index 0000000..fe47d94 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, console + +######## console appender ######## +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties new file mode 100644 index 0000000..d65f9be --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties @@ -0,0 +1,5 @@ +# Generated by Apache Ambari. Thu Jun 18 00:59:57 2015 + +atlas.http.authentication.enabled=false +atlas.http.authentication.type=simple + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh new file mode 100644 index 0000000..fd99df0 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh @@ -0,0 +1,34 @@ + + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR=/usr/hdp/current/hive-client/conf + +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar +fi + +export METASTORE_PORT=9083
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template new file mode 100644 index 0000000..91a2905 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hive and Hadoop environment variables here. These variables can be used +# to control the execution of Hive. It should be used by admins to configure +# the Hive installation (so that users do not have to set environment variables +# or set command line parameters to get correct behavior). +# +# The hive service being invoked (CLI/HWI etc.) is available via the environment +# variable SERVICE + + +# Hive Client memory usage can be an issue if a large number of clients +# are running at the same time. The flags below have been useful in +# reducing memory usage: +# +# if [ "$SERVICE" = "cli" ]; then +# if [ -z "$DEBUG" ]; then +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" +# else +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" +# fi +# fi + +# The heap size of the jvm stared by hive shell script can be controlled via: +# +# export HADOOP_HEAPSIZE=1024 +# +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +# HADOOP_HOME=${bin}/../../hadoop + +# Hive Configuration Directory can be controlled by: +# export HIVE_CONF_DIR= + +# Folder containing extra ibraries required for hive compilation/execution can be controlled by: +# export HIVE_AUX_JARS_PATH= diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties new file mode 100644 index 0000000..5b59a3f --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties @@ -0,0 +1,79 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger=INFO,FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties new file mode 100644 index 0000000..7861ba5 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties @@ -0,0 +1,89 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hive.log.threshold=ALL +hive.root.logger=INFO,DRFA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.log.file=hive.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${hive.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. +# +# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,DRFA +log4j.category.Datastore=ERROR,DRFA +log4j.category.Datastore.Schema=ERROR,DRFA +log4j.category.JPOX.Datastore=ERROR,DRFA +log4j.category.JPOX.Plugin=ERROR,DRFA +log4j.category.JPOX.MetaData=ERROR,DRFA +log4j.category.JPOX.Query=ERROR,DRFA +log4j.category.JPOX.General=ERROR,DRFA +log4j.category.JPOX.Enhancer=ERROR,DRFA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
\ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml new file mode 100644 index 0000000..a3c34e3 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml @@ -0,0 +1,789 @@ +<!--Thu Jun 18 00:59:56 2015--> + <configuration> + + <property> + <name>ambari.hive.db.schema.name</name> + <value>hive</value> + </property> + + <property> + <name>datanucleus.autoCreateSchema</name> + <value>false</value> + </property> + + <property> + <name>datanucleus.cache.level2.type</name> + <value>none</value> + </property> + + <property> + <name>hive.auto.convert.join</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.join.noconditionaltask</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.join.noconditionaltask.size</name> + <value>894784504</value> + </property> + + <property> + <name>hive.auto.convert.sortmerge.join</name> + <value>true</value> + </property> + + <property> + <name>hive.auto.convert.sortmerge.join.to.mapjoin</name> + <value>false</value> + </property> + + <property> + <name>hive.cbo.enable</name> + <value>true</value> + </property> + + <property> + <name>hive.cli.print.header</name> + <value>false</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.class</name> + <value>org.apache.hadoop.hive.thrift.ZooKeeperTokenStore</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.zookeeper.connectString</name> + <value>TODO_REPLACE_HOSTNAME:2181</value> + </property> + + <property> + <name>hive.cluster.delegation.token.store.zookeeper.znode</name> + <value>/hive/cluster/delegation</value> + </property> + + <property> + <name>hive.cluster.name</name> + <value>companion</value> + </property> + + <property> + <name>hive.compactor.abortedtxn.threshold</name> + <value>1000</value> + </property> + + <property> + <name>hive.compactor.check.interval</name> + <value>300L</value> + </property> + + <property> + <name>hive.compactor.delta.num.threshold</name> + <value>10</value> + </property> + + <property> + <name>hive.compactor.delta.pct.threshold</name> + <value>0.1f</value> + </property> + + <property> + <name>hive.compactor.initiator.on</name> + <value>false</value> + </property> + + <property> + <name>hive.compactor.worker.threads</name> + <value>0</value> + </property> + + <property> + <name>hive.compactor.worker.timeout</name> + <value>86400L</value> + </property> + + <property> + <name>hive.compute.query.using.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.conf.restricted.list</name> + <value>hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role</value> + </property> + + <property> + <name>hive.convert.join.bucket.mapjoin.tez</name> + <value>false</value> + </property> + + <property> + <name>hive.default.fileformat</name> + <value>TextFile</value> + </property> + + <property> + <name>hive.enforce.bucketing</name> + <value>false</value> + </property> + + <property> + <name>hive.enforce.sorting</name> + <value>true</value> + </property> + + <property> + <name>hive.enforce.sortmergebucketmapjoin</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.compress.intermediate</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.compress.output</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.dynamic.partition</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.dynamic.partition.mode</name> + <value>strict</value> + </property> + + <property> + <name>hive.exec.failure.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> + </property> + + <property> + <name>hive.exec.max.created.files</name> + <value>100000</value> + </property> + + <property> + <name>hive.exec.max.dynamic.partitions</name> + <value>5000</value> + </property> + + <property> + <name>hive.exec.max.dynamic.partitions.pernode</name> + <value>2000</value> + </property> + + <property> + <name>hive.exec.orc.compression.strategy</name> + <value>SPEED</value> + </property> + + <property> + <name>hive.exec.orc.default.compress</name> + <value>ZLIB</value> + </property> + + <property> + <name>hive.exec.orc.default.stripe.size</name> + <value>67108864</value> + </property> + + <property> + <name>hive.exec.orc.encoding.strategy</name> + <value>SPEED</value> + </property> + + <property> + <name>hive.exec.parallel</name> + <value>false</value> + </property> + + <property> + <name>hive.exec.parallel.thread.number</name> + <value>8</value> + </property> + + <property> + <name>hive.exec.post.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook, org.apache.atlas.hive.hook.HiveHook</value> + </property> + + <property> + <name>hive.exec.pre.hooks</name> + <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value> + </property> + + <property> + <name>hive.exec.reducers.bytes.per.reducer</name> + <value>67108864</value> + </property> + + <property> + <name>hive.exec.reducers.max</name> + <value>1009</value> + </property> + + <property> + <name>hive.exec.scratchdir</name> + <value>/tmp/hive</value> + </property> + + <property> + <name>hive.exec.submit.local.task.via.child</name> + <value>true</value> + </property> + + <property> + <name>hive.exec.submitviachild</name> + <value>false</value> + </property> + + <property> + <name>hive.execution.engine</name> + <value>tez</value> + </property> + + <property> + <name>hive.fetch.task.aggr</name> + <value>false</value> + </property> + + <property> + <name>hive.fetch.task.conversion</name> + <value>more</value> + </property> + + <property> + <name>hive.fetch.task.conversion.threshold</name> + <value>1073741824</value> + </property> + + <property> + <name>hive.hook.dgi.url</name> + <value>http://TODO_REPLACE_HOSTNAME:21000</value> + </property> + + <property> + <name>hive.limit.optimize.enable</name> + <value>true</value> + </property> + + <property> + <name>hive.limit.pushdown.memory.usage</name> + <value>0.04</value> + </property> + + <property> + <name>hive.map.aggr</name> + <value>true</value> + </property> + + <property> + <name>hive.map.aggr.hash.force.flush.memory.threshold</name> + <value>0.9</value> + </property> + + <property> + <name>hive.map.aggr.hash.min.reduction</name> + <value>0.5</value> + </property> + + <property> + <name>hive.map.aggr.hash.percentmemory</name> + <value>0.5</value> + </property> + + <property> + <name>hive.mapjoin.bucket.cache.size</name> + <value>10000</value> + </property> + + <property> + <name>hive.mapjoin.optimized.hashtable</name> + <value>true</value> + </property> + + <property> + <name>hive.mapred.reduce.tasks.speculative.execution</name> + <value>false</value> + </property> + + <property> + <name>hive.merge.mapfiles</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.mapredfiles</name> + <value>false</value> + </property> + + <property> + <name>hive.merge.orcfile.stripe.level</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.rcfile.block.level</name> + <value>true</value> + </property> + + <property> + <name>hive.merge.size.per.task</name> + <value>256000000</value> + </property> + + <property> + <name>hive.merge.smallfiles.avgsize</name> + <value>16000000</value> + </property> + + <property> + <name>hive.merge.tezfiles</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.authorization.storage.checks</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.cache.pinobjtypes</name> + <value>Table,Database,Type,FieldSchema,Order</value> + </property> + + <property> + <name>hive.metastore.client.connect.retry.delay</name> + <value>5s</value> + </property> + + <property> + <name>hive.metastore.client.socket.timeout</name> + <value>1800s</value> + </property> + + <property> + <name>hive.metastore.connect.retries</name> + <value>24</value> + </property> + + <property> + <name>hive.metastore.execute.setugi</name> + <value>true</value> + </property> + + <property> + <name>hive.metastore.failure.retries</name> + <value>24</value> + </property> + + <property> + <name>hive.metastore.kerberos.keytab.file</name> + <value>/etc/security/keytabs/hive.service.keytab</value> + </property> + + <property> + <name>hive.metastore.kerberos.principal</name> + <value>hive/_HOST@EXAMPLE.COM</value> + </property> + + <property> + <name>hive.metastore.pre.event.listeners</name> + <value>org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener</value> + </property> + + <property> + <name>hive.metastore.sasl.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.metastore.server.max.threads</name> + <value>100000</value> + </property> + + <property> + <name>hive.metastore.uris</name> + <value>thrift://TODO_REPLACE_HOSTNAME:9083</value> + </property> + + <property> + <name>hive.metastore.warehouse.dir</name> + <value>/apps/hive/warehouse</value> + </property> + + <property> + <name>hive.optimize.bucketmapjoin</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.bucketmapjoin.sortedmerge</name> + <value>false</value> + </property> + + <property> + <name>hive.optimize.constant.propagation</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.index.filter</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.metadataonly</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.null.scan</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.reducededuplication</name> + <value>true</value> + </property> + + <property> + <name>hive.optimize.reducededuplication.min.reducer</name> + <value>4</value> + </property> + + <property> + <name>hive.optimize.sort.dynamic.partition</name> + <value>false</value> + </property> + + <property> + <name>hive.orc.compute.splits.num.threads</name> + <value>10</value> + </property> + + <property> + <name>hive.orc.splits.include.file.footer</name> + <value>false</value> + </property> + + <property> + <name>hive.prewarm.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.prewarm.numcontainers</name> + <value>3</value> + </property> + + <property> + <name>hive.security.authenticator.manager</name> + <value>org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator</value> + </property> + + <property> + <name>hive.security.authorization.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.security.authorization.manager</name> + <value>org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory</value> + </property> + + <property> + <name>hive.security.metastore.authenticator.manager</name> + <value>org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator</value> + </property> + + <property> + <name>hive.security.metastore.authorization.auth.reads</name> + <value>true</value> + </property> + + <property> + <name>hive.security.metastore.authorization.manager</name> + <value>org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider</value> + </property> + + <property> + <name>hive.server2.allow.user.substitution</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.authentication</name> + <value>NONE</value> + </property> + + <property> + <name>hive.server2.authentication.spnego.keytab</name> + <value>HTTP/_HOST@EXAMPLE.COM</value> + </property> + + <property> + <name>hive.server2.authentication.spnego.principal</name> + <value>/etc/security/keytabs/spnego.service.keytab</value> + </property> + + <property> + <name>hive.server2.enable.doAs</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.logging.operation.enabled</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.logging.operation.log.location</name> + <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value> + </property> + + <property> + <name>hive.server2.support.dynamic.service.discovery</name> + <value>true</value> + </property> + + <property> + <name>hive.server2.table.type.mapping</name> + <value>CLASSIC</value> + </property> + + <property> + <name>hive.server2.tez.default.queues</name> + <value>default</value> + </property> + + <property> + <name>hive.server2.tez.initialize.default.sessions</name> + <value>false</value> + </property> + + <property> + <name>hive.server2.tez.sessions.per.default.queue</name> + <value>1</value> + </property> + + <property> + <name>hive.server2.thrift.http.path</name> + <value>cliservice</value> + </property> + + <property> + <name>hive.server2.thrift.http.port</name> + <value>10001</value> + </property> + + <property> + <name>hive.server2.thrift.max.worker.threads</name> + <value>500</value> + </property> + + <property> + <name>hive.server2.thrift.port</name> + <value>10000</value> + </property> + + <property> + <name>hive.server2.thrift.sasl.qop</name> + <value>auth</value> + </property> + + <property> + <name>hive.server2.transport.mode</name> + <value>binary</value> + </property> + + <property> + <name>hive.server2.use.SSL</name> + <value>false</value> + </property> + + <property> + <name>hive.server2.zookeeper.namespace</name> + <value>hiveserver2</value> + </property> + + <property> + <name>hive.smbjoin.cache.rows</name> + <value>10000</value> + </property> + + <property> + <name>hive.stats.autogather</name> + <value>true</value> + </property> + + <property> + <name>hive.stats.dbclass</name> + <value>fs</value> + </property> + + <property> + <name>hive.stats.fetch.column.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.stats.fetch.partition.stats</name> + <value>true</value> + </property> + + <property> + <name>hive.support.concurrency</name> + <value>false</value> + </property> + + <property> + <name>hive.tez.auto.reducer.parallelism</name> + <value>true</value> + </property> + + <property> + <name>hive.tez.container.size</name> + <value>2560</value> + </property> + + <property> + <name>hive.tez.cpu.vcores</name> + <value>-1</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning</name> + <value>true</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning.max.data.size</name> + <value>104857600</value> + </property> + + <property> + <name>hive.tez.dynamic.partition.pruning.max.event.size</name> + <value>1048576</value> + </property> + + <property> + <name>hive.tez.input.format</name> + <value>org.apache.hadoop.hive.ql.io.HiveInputFormat</value> + </property> + + <property> + <name>hive.tez.java.opts</name> + <value>-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps</value> + </property> + + <property> + <name>hive.tez.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>hive.tez.max.partition.factor</name> + <value>2.0</value> + </property> + + <property> + <name>hive.tez.min.partition.factor</name> + <value>0.25</value> + </property> + + <property> + <name>hive.tez.smb.number.waves</name> + <value>0.5</value> + </property> + + <property> + <name>hive.txn.manager</name> + <value>org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager</value> + </property> + + <property> + <name>hive.txn.max.open.batch</name> + <value>1000</value> + </property> + + <property> + <name>hive.txn.timeout</name> + <value>300</value> + </property> + + <property> + <name>hive.user.install.directory</name> + <value>/user/</value> + </property> + + <property> + <name>hive.vectorized.execution.enabled</name> + <value>true</value> + </property> + + <property> + <name>hive.vectorized.execution.reduce.enabled</name> + <value>false</value> + </property> + + <property> + <name>hive.vectorized.groupby.checkinterval</name> + <value>4096</value> + </property> + + <property> + <name>hive.vectorized.groupby.flush.percent</name> + <value>0.1</value> + </property> + + <property> + <name>hive.vectorized.groupby.maxentries</name> + <value>100000</value> + </property> + + <property> + <name>hive.zookeeper.client.port</name> + <value>2181</value> + </property> + + <property> + <name>hive.zookeeper.namespace</name> + <value>hive_zookeeper_namespace</value> + </property> + + <property> + <name>hive.zookeeper.quorum</name> + <value>TODO_REPLACE_HOSTNAME:2181</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionDriverName</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:mysql://TODO_REPLACE_HOSTNAME/hive?createDatabaseIfNotExist=true</value> + </property> + + <property> + <name>javax.jdo.option.ConnectionUserName</name> + <value>hive</value> + </property> + + <property> + <name>mapreduce.input.fileinputformat.split.minsize</name> + <value>1000000</value> + </property> + + </configuration> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml new file mode 100644 index 0000000..bda842a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml @@ -0,0 +1,37 @@ + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<!--This file is used by grapes to download dependencies from a maven repository. + This is just a template and can be edited to add more repositories. +--> + +<ivysettings> + <!--name of the defaultResolver should always be 'downloadGrapes'. --> + <settings defaultResolver="downloadGrapes"/> + <resolvers> + <!-- more resolvers can be added here --> + <chain name="downloadGrapes"> + <!-- This resolver uses ibiblio to find artifacts, compatible with maven2 repository --> + <ibiblio name="central" m2compatible="true"/> + <!-- File resolver to add jars from the local system. --> + <filesystem name="test" checkmodified="true"> + <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar" /> + </filesystem> + </chain> + </resolvers> +</ivysettings> diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml new file mode 100644 index 0000000..6c71cae --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml @@ -0,0 +1,224 @@ +<!--Thu Jun 18 04:07:38 2015--> + <configuration> + + <property> + <name>mapreduce.admin.map.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.reduce.child.java.opts</name> + <value>-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>mapreduce.admin.user.env</name> + <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value> + </property> + + <property> + <name>mapreduce.am.max-attempts</name> + <value>2</value> + </property> + + <property> + <name>mapreduce.application.classpath</name> + <value>$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value> + </property> + + <property> + <name>mapreduce.application.framework.path</name> + <value>/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework</value> + </property> + + <property> + <name>mapreduce.cluster.administrators</name> + <value> hadoop</value> + </property> + + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + + <property> + <name>mapreduce.job.counters.max</name> + <value>130</value> + </property> + + <property> + <name>mapreduce.job.emit-timeline-data</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.job.reduce.slowstart.completedmaps</name> + <value>0.05</value> + </property> + + <property> + <name>mapreduce.jobhistory.address</name> + <value>TODO_REPLACE_HOSTNAME:10020</value> + </property> + + <property> + <name>mapreduce.jobhistory.bind-host</name> + <value>0.0.0.0</value> + </property> + + <property> + <name>mapreduce.jobhistory.done-dir</name> + <value>/mr-history/done</value> + </property> + + <property> + <name>mapreduce.jobhistory.intermediate-done-dir</name> + <value>/mr-history/tmp</value> + </property> + + <property> + <name>mapreduce.jobhistory.webapp.address</name> + <value>TODO_REPLACE_HOSTNAME:19888</value> + </property> + + <property> + <name>mapreduce.map.java.opts</name> + <value>-Xmx2048m</value> + </property> + + <property> + <name>mapreduce.map.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.map.memory.mb</name> + <value>2560</value> + </property> + + <property> + <name>mapreduce.map.output.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.map.sort.spill.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.map.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.output.fileoutputformat.compress.type</name> + <value>BLOCK</value> + </property> + + <property> + <name>mapreduce.reduce.input.buffer.percent</name> + <value>0.0</value> + </property> + + <property> + <name>mapreduce.reduce.java.opts</name> + <value>-Xmx4096m</value> + </property> + + <property> + <name>mapreduce.reduce.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>mapreduce.reduce.memory.mb</name> + <value>5120</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.enabled</name> + <value>1</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.interval-ms</name> + <value>1000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.fetch.retry.timeout-ms</name> + <value>30000</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.input.buffer.percent</name> + <value>0.7</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.merge.percent</name> + <value>0.66</value> + </property> + + <property> + <name>mapreduce.reduce.shuffle.parallelcopies</name> + <value>30</value> + </property> + + <property> + <name>mapreduce.reduce.speculative</name> + <value>false</value> + </property> + + <property> + <name>mapreduce.shuffle.port</name> + <value>13562</value> + </property> + + <property> + <name>mapreduce.task.io.sort.factor</name> + <value>100</value> + </property> + + <property> + <name>mapreduce.task.io.sort.mb</name> + <value>1433</value> + </property> + + <property> + <name>mapreduce.task.timeout</name> + <value>300000</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.admin-command-opts</name> + <value>-Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.command-opts</name> + <value>-Xmx2048m -Dhdp.version=${hdp.version}</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.log.level</name> + <value>INFO</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.resource.mb</name> + <value>2560</value> + </property> + + <property> + <name>yarn.app.mapreduce.am.staging-dir</name> + <value>/user</value> + </property> + + </configuration> |