From 39fd75bdb79ac63a05dadda0ba32792e49783031 Mon Sep 17 00:00:00 2001 From: lj1412 Date: Tue, 14 Feb 2017 15:10:55 +0000 Subject: Init dcae.apod.cdap Change-Id: I0d1f1ba04f8ba7a2a276e4e382eb07ae9d6a7652 Signed-off-by: lj1412 --- .gitreview | 4 + LICENSE.txt | 19 + README.md | 91 +++ cdap3vm/config/cdap-config-template/cdap-env.sh | 26 + cdap3vm/config/cdap-config-template/cdap-site.xml | 61 ++ .../config/cdap-config-template/common/common.sh | 309 ++++++++ .../cdap-config-template/logback-container.xml | 70 ++ cdap3vm/config/cdap-config-template/logback.xml | 81 +++ cdap3vm/config/hadoop-cluster-conf-file.sh | 131 ++++ .../core_hadoop/capacity-scheduler.xml | 79 +++ .../core_hadoop/container-executor.cfg | 23 + .../core_hadoop/core-site.xml | 147 ++++ .../core_hadoop/hadoop-env.sh | 125 ++++ .../core_hadoop/hadoop-policy.xml | 59 ++ .../core_hadoop/hdfs-site.xml | 281 ++++++++ .../core_hadoop/health_check | 64 ++ .../core_hadoop/kms-acls.xml | 135 ++++ .../core_hadoop/kms-env.sh | 55 ++ .../core_hadoop/kms-site.xml | 173 +++++ .../core_hadoop/log4j.properties | 225 ++++++ .../core_hadoop/mapred-env.sh | 14 + .../core_hadoop/mapred-site.xml | 224 ++++++ .../core_hadoop/ssl-client.xml | 39 + .../core_hadoop/ssl-server.xml | 44 ++ .../core_hadoop/taskcontroller.cfg | 21 + .../core_hadoop/yarn-env.sh | 119 ++++ .../core_hadoop/yarn-site.xml | 527 ++++++++++++++ .../core_hadoop/yarn.exclude | 1 + .../hbase/core-site.xml | 147 ++++ .../hbase/hadoop-metrics2-hbase.properties | 50 ++ .../hbase/hbase-env.cmd | 87 +++ .../hbase/hbase-env.sh | 65 ++ .../hbase/hbase-policy.xml | 19 + .../hbase/hbase-site.xml | 234 ++++++ .../hbase/hdfs-site.xml | 281 ++++++++ .../hbase/log4j.properties | 111 +++ .../hbase/regionservers | 3 + .../hive/beeline-log4j.properties.template | 24 + .../hive/client.properties | 5 + .../hive/hive-env.sh | 34 + .../hive/hive-env.sh.template | 54 ++ .../hive/hive-exec-log4j.properties | 79 +++ .../hive/hive-log4j.properties | 89 +++ .../hive/hive-site.xml | 789 +++++++++++++++++++++ .../hive/ivysettings.xml | 37 + .../hive/mapred-site.xml | 224 ++++++ .../kafka/consumer.properties | 29 + .../kafka/kafka-env.sh | 19 + .../kafka/log4j.properties | 84 +++ .../kafka/producer.properties | 53 ++ .../kafka/server.properties | 70 ++ .../kafka/test-log4j.properties | 66 ++ .../kafka/tools-log4j.properties | 20 + .../kafka/zookeeper.properties | 20 + .../zookeeper/configuration.xsl | 25 + .../zookeeper/log4j.properties | 70 ++ .../zookeeper/zoo.cfg | 37 + .../zookeeper/zoo_sample.cfg | 28 + .../zookeeper/zookeeper-env.cmd | 28 + .../zookeeper/zookeeper-env.sh | 9 + cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl | 11 + cdap3vm/config/hostname-xref.txt.test | 3 + .../config/services-setup-templates/cdap-setup.sh | 6 + .../config/services-setup-templates/cdap-start.sh | 11 + .../services-setup-templates/datanode-start.sh | 4 + .../services-setup-templates/hbase-master-start.sh | 3 + .../hbase-regionserver-start.sh | 5 + .../job-history-setup-01-as-root.sh | 4 + .../job-history-setup-02-as-hdfs.sh | 16 + .../services-setup-templates/namenode-setup.sh | 34 + .../services-setup-templates/node-manager-start.sh | 4 + .../resource-manager-start.sh | 4 + .../services-setup-templates/service-setup.sh | 10 + .../services-setup-templates/zookeeper-start.sh | 12 + .../zookeeper-zkfc-start.sh | 3 + .../services-templates/boot-time-cdap-vm-N0.sh | 52 ++ .../services-templates/boot-time-cdap-vm-N1.sh | 56 ++ .../services-templates/boot-time-cdap-vm-N2.sh | 50 ++ cdap3vm/config/services-templates/cdap-vm-services | 46 ++ cdap3vm/config/services-templates/cdap.sh | 36 + cdap3vm/config/services-templates/datanode.sh | 34 + cdap3vm/config/services-templates/hbase-master.sh | 34 + .../services-templates/hbase-regionserver.sh | 32 + .../services-templates/job-history-server.sh | 35 + cdap3vm/config/services-templates/namenode.sh | 32 + cdap3vm/config/services-templates/node-manager.sh | 32 + .../config/services-templates/resource-manager.sh | 35 + .../services-templates/secondary-namenode.sh | 33 + cdap3vm/config/services-templates/service-start.sh | 27 + .../config/services-templates/utility-scripts.sh | 41 ++ .../config/services-templates/zookeeper-zkfc.sh | 33 + cdap3vm/config/services-templates/zookeeper.sh | 35 + .../install-steps/01-generate-host-ids-configs.sh | 145 ++++ cdap3vm/install-steps/02-user-creation.sh | 43 ++ cdap3vm/install-steps/04-folder-creation.sh | 71 ++ cdap3vm/install-steps/install-cdap-pkgs.sh | 49 ++ .../install-steps/install-hortonworks-hadoop.sh | 61 ++ .../install-steps/utils/boot-time-cdap-vm-N0.sh | 33 + .../install-steps/utils/boot-time-cdap-vm-N1.sh | 35 + .../install-steps/utils/boot-time-cdap-vm-N2.sh | 33 + cdap3vm/install-steps/utils/cdap-nodetype-N0.sh | 29 + cdap3vm/install-steps/utils/cdap-nodetype-N1.sh | 34 + cdap3vm/install-steps/utils/cdap-nodetype-N2.sh | 29 + cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh | 30 + cdap3vm/install-steps/utils/cdap-vm-services | 42 ++ .../install-steps/utils/folder-creation-utils.sh | 241 +++++++ cdap3vm/install-steps/utils/generate-hosts.sh | 38 + cdap3vm/install-steps/utils/replace-kv.py | 63 ++ cdap3vm/install.sh | 32 + cdap3vm/pkgs/services-setup/cdap-setup.sh | 34 + cdap3vm/pkgs/services-setup/datanode-start.sh | 22 + cdap3vm/pkgs/services-setup/hbase-master-start.sh | 21 + .../services-setup/hbase-regionserver-start.sh | 23 + .../services-setup/job-history-setup-01-as-root.sh | 22 + .../services-setup/job-history-setup-02-as-hdfs.sh | 34 + cdap3vm/pkgs/services-setup/namenode-setup.sh | 44 ++ cdap3vm/pkgs/services-setup/node-manager-start.sh | 22 + .../pkgs/services-setup/resource-manager-start.sh | 22 + cdap3vm/pkgs/services-setup/service-setup.sh | 28 + cdap3vm/pkgs/services-setup/zookeeper-start.sh | 30 + cdap3vm/pkgs/services/datanode.sh | 22 + cdap3vm/pkgs/services/hbase-master.sh | 21 + cdap3vm/pkgs/services/hbase-regionserver.sh | 23 + cdap3vm/pkgs/services/job-history-server.sh | 22 + cdap3vm/pkgs/services/namenode.sh | 22 + cdap3vm/pkgs/services/node-manager.sh | 22 + cdap3vm/pkgs/services/resource-manager.sh | 22 + cdap3vm/pkgs/services/secondary-namenode.sh | 22 + cdap3vm/pkgs/services/service-start.sh | 27 + cdap3vm/pkgs/services/zookeeper.sh | 35 + cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf | 34 + cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh | 48 ++ 132 files changed, 8440 insertions(+) create mode 100644 .gitreview create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 cdap3vm/config/cdap-config-template/cdap-env.sh create mode 100644 cdap3vm/config/cdap-config-template/cdap-site.xml create mode 100755 cdap3vm/config/cdap-config-template/common/common.sh create mode 100644 cdap3vm/config/cdap-config-template/logback-container.xml create mode 100644 cdap3vm/config/cdap-config-template/logback.xml create mode 100644 cdap3vm/config/hadoop-cluster-conf-file.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg create mode 100755 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/client.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd create mode 100644 cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh create mode 100644 cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl create mode 100644 cdap3vm/config/hostname-xref.txt.test create mode 100644 cdap3vm/config/services-setup-templates/cdap-setup.sh create mode 100644 cdap3vm/config/services-setup-templates/cdap-start.sh create mode 100644 cdap3vm/config/services-setup-templates/datanode-start.sh create mode 100644 cdap3vm/config/services-setup-templates/hbase-master-start.sh create mode 100644 cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh create mode 100644 cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh create mode 100644 cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh create mode 100644 cdap3vm/config/services-setup-templates/namenode-setup.sh create mode 100644 cdap3vm/config/services-setup-templates/node-manager-start.sh create mode 100644 cdap3vm/config/services-setup-templates/resource-manager-start.sh create mode 100644 cdap3vm/config/services-setup-templates/service-setup.sh create mode 100644 cdap3vm/config/services-setup-templates/zookeeper-start.sh create mode 100644 cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh create mode 100644 cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh create mode 100644 cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh create mode 100644 cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh create mode 100644 cdap3vm/config/services-templates/cdap-vm-services create mode 100644 cdap3vm/config/services-templates/cdap.sh create mode 100644 cdap3vm/config/services-templates/datanode.sh create mode 100644 cdap3vm/config/services-templates/hbase-master.sh create mode 100644 cdap3vm/config/services-templates/hbase-regionserver.sh create mode 100644 cdap3vm/config/services-templates/job-history-server.sh create mode 100644 cdap3vm/config/services-templates/namenode.sh create mode 100644 cdap3vm/config/services-templates/node-manager.sh create mode 100644 cdap3vm/config/services-templates/resource-manager.sh create mode 100644 cdap3vm/config/services-templates/secondary-namenode.sh create mode 100644 cdap3vm/config/services-templates/service-start.sh create mode 100644 cdap3vm/config/services-templates/utility-scripts.sh create mode 100644 cdap3vm/config/services-templates/zookeeper-zkfc.sh create mode 100644 cdap3vm/config/services-templates/zookeeper.sh create mode 100644 cdap3vm/install-steps/01-generate-host-ids-configs.sh create mode 100644 cdap3vm/install-steps/02-user-creation.sh create mode 100644 cdap3vm/install-steps/04-folder-creation.sh create mode 100755 cdap3vm/install-steps/install-cdap-pkgs.sh create mode 100755 cdap3vm/install-steps/install-hortonworks-hadoop.sh create mode 100644 cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh create mode 100644 cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh create mode 100644 cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh create mode 100644 cdap3vm/install-steps/utils/cdap-nodetype-N0.sh create mode 100644 cdap3vm/install-steps/utils/cdap-nodetype-N1.sh create mode 100644 cdap3vm/install-steps/utils/cdap-nodetype-N2.sh create mode 100644 cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh create mode 100644 cdap3vm/install-steps/utils/cdap-vm-services create mode 100644 cdap3vm/install-steps/utils/folder-creation-utils.sh create mode 100644 cdap3vm/install-steps/utils/generate-hosts.sh create mode 100644 cdap3vm/install-steps/utils/replace-kv.py create mode 100644 cdap3vm/install.sh create mode 100644 cdap3vm/pkgs/services-setup/cdap-setup.sh create mode 100644 cdap3vm/pkgs/services-setup/datanode-start.sh create mode 100644 cdap3vm/pkgs/services-setup/hbase-master-start.sh create mode 100644 cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh create mode 100644 cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh create mode 100644 cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh create mode 100644 cdap3vm/pkgs/services-setup/namenode-setup.sh create mode 100644 cdap3vm/pkgs/services-setup/node-manager-start.sh create mode 100644 cdap3vm/pkgs/services-setup/resource-manager-start.sh create mode 100644 cdap3vm/pkgs/services-setup/service-setup.sh create mode 100644 cdap3vm/pkgs/services-setup/zookeeper-start.sh create mode 100644 cdap3vm/pkgs/services/datanode.sh create mode 100644 cdap3vm/pkgs/services/hbase-master.sh create mode 100644 cdap3vm/pkgs/services/hbase-regionserver.sh create mode 100644 cdap3vm/pkgs/services/job-history-server.sh create mode 100644 cdap3vm/pkgs/services/namenode.sh create mode 100644 cdap3vm/pkgs/services/node-manager.sh create mode 100644 cdap3vm/pkgs/services/resource-manager.sh create mode 100644 cdap3vm/pkgs/services/secondary-namenode.sh create mode 100644 cdap3vm/pkgs/services/service-start.sh create mode 100644 cdap3vm/pkgs/services/zookeeper.sh create mode 100644 cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf create mode 100644 cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..c7a4d8f --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.openecomp.org +port=29418 +project=dcae/apod/cdap.git diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..7255bc3 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +/* + * ============LICENSE_START========================================== + * =================================================================== + * Copyright © 2017 AT&T Intellectual Property. All rights reserved. + * =================================================================== + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END============================================ + * ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + */ diff --git a/README.md b/README.md new file mode 100644 index 0000000..463a710 --- /dev/null +++ b/README.md @@ -0,0 +1,91 @@ +Building the DCAE CDAP Cluster Package +======================================== + +This is the repository for the CDAP cluster that will be used for running analytics. Currently this repository contains the scripts and packages to create a DCAE CDAP small hadoop package, which is a 3 node hadoop environment that CDAP will be installed on. + +### Build Instructions + +There is no source code to build. This repository contains the scripts and packages used to install the CDAP 3 VM environment. + +### Debian Packaging + +The contents of this repository will be bundled into a Debian package and sent to the OpenECOMP Nexus server where +it will be picked up and installed by the DCAE Controller. Following is the process to +create the Debian package and upload it to the OpenECOMP Nexus distribution server. + + +#### Extract the contents to package +``` +git clone git@:dcae-apod/dcae-org.openecomp.cdapcluster.git +``` + +#### Set up the packaging environment +1. Extract the build tools and config file +The build tools and configuration files for all of APOD are in the dcae-apod-buildtools repository which is part of the dcae-apod Project. +``` +$ git clone ssh://git@:dcae-apod/dcae-apod-buildtools.git +``` +From this repo you will need: + - The package script from the scripts directory + - The package-cdap3vm.json file from the configs directory + +2. Set up the packaging environment +The packaging script requires directories named stage and output. The stage directory will contain all the directories and files that will packaged. The output directory is where the Debian file will be put. + +The package-cdap3vm.json configuration file contains the parameters that the package script needs in order to create the Debian package. + +NOTE: The user should only set the variables between the "User sets these" block. The Debian file MUST have a specific name in order for the DCAE controller to find it an install it. +``` +########### User sets these ############### +WORK_DIR=/path/to/your/work/dir/ # This is the path of where you will do your packaging +BUILD_TOOLS_DIR=/path/to/dcae-apod-buildtools # This is the path to dcae-apod-buildtools directory +CDAP3VM_SOURCE_DIR=/path/to/source/dir # This is the where you have checked out dcae-org.openecomp.cdapcluster +OPENECOMP_NEXUS_RAW=https://xx.xx.xx:8443/repository/raw # This is the URL of the OpenECOMP raw repository in the Nexus distribution server +OPENECOMP_NEXUS_USER=user-name +OPENECOMP_NEXUS_PASSWORD=password +########### End of user set variables ############### + +STAGE_DIR=${WORK_DIR}/package +OUTPUT_DIR=${WORK_DIR}/package/output + +DATE_STAMP="$(date +"%Y%m%d%H%M%S")" +PACKAGE_BUILD_NUMBER="LATEST" +PACKAGE_NAME_APPLICATION=$(cat ${BUILD_TOOLS_DIR}/dcae-apod-buildtools/configs/package-cdap3vm.json | python -c 'import json,sys;print json.load(sys.stdin)["applicationName"]') +PACKAGE_NAME_VERSION=$(cat ${BUILD_TOOLS_DIR}/dcae-apod-buildtools/configs/package-cdap3vm.json | python -c 'import json,sys;print json.load(sys.stdin)["version"]') +PACKAGE_GROUP_ID=$(cat ${BUILD_TOOLS_DIR}/dcae-apod-buildtools/configs/package-cdap3vm.json | python -c 'import json,sys;print json.load(sys.stdin)["groupId"]') + +OUTPUT_FILE=${PACKAGE_NAME_APPLICATION}"_"${PACKAGE_NAME_VERSION}"-"${PACKAGE_BUILD_NUMBER}".deb" +OUTPUT_FILE_DATE_STAMPED=${PACKAGE_NAME_APPLICATION}"_"${PACKAGE_NAME_VERSION}"-"${DATE_STAMP}".deb" + +rm -rf ${STAGE_DIR} +rm -rf ${OUTPUT_DIR} +mkdir -p ${STAGE_DIR}/stage/opt/app/dcae-cdap-small-hadoop +mkdir -p ${OUTPUT_DIR} + +echo "Copying jar file to stage" +cp -R ${CDAP3VM_SOURCE_DIR}/CDAP3VM ${STAGE_DIR}/stage/opt/app/dcae-cdap-small-hadoop + +echo "Copying json config file to stage" +cp ${BUILD_TOOLS_DIR}/dcae-apod-buildtools/configs/package-cdap3vm.json ${STAGE_DIR}/package.json +``` +#### Run the script to create the Debian package +The package script will package what is in the stage directory as well as create a debian control file, change log, copyright file, md5checksums file and a postint file with any post install steps that need to be run as part of the installation. + +The package script will put the Debian file that is created in ${OUTPUT_DIR} +``` +echo "Creating debian package" +${BUILD_DIR}/dcae-apod-buildtools/scripts/package -b debian -d ${STAGE_DIR} -o ${OUTPUT_DIR} -y package.json -B ${PACKAGE_BUILD_NUMBER} -v +``` +#### Push the package to the OpenECOMP Nexus distribution server +

The Debian package that is created needs to be pushed up to the OpenECOMP Nexus distribution server where the DCAE Controller will pick it up an install it. +

The controller needs the debian packaged named dcae-cdap-small-hadoop_17.01.0-LATEST.deb so it can find and deploy it. In order to have a copy of each file built a copy of dcae-cdap-small-hadoop_17.01.0-LATEST.deb will be made and it will have a date stamp and build number on it. For example: dcae-cdap-small-hadoop_17.01.0-YYYYMMDDHHMMSS-XXX.deb Both files will then be uploaded to the repository. + +``` +cp ${OUTPUT_DIR}/${OUTPUT_FILE} ${OUTPUT_DIR}/${OUTPUT_FILE_DATE_STAMPED} + +SEND_TO=${OPENECOMP_NEXUS_RAW}"/org.openecomp.dcae/deb-snapshots/"${PACKAGE_GROUP_ID}"/"${OUTPUT_FILE} +curl -vk --user ${OPENECOMP_NEXUS_USER}:${OPENECOMP_NEXUS_PASSWORD} --upload-file ${OUTPUT_DIR}/${OUTPUT_FILE} ${SEND_TO} + +SEND_TO=${OPENECOMP_NEXUS_RAW}"/org.openecomp.dcae/deb-snapshots/"${PACKAGE_GROUP_ID}"/"${OUTPUT_FILE_DATE_STAMPED} +curl -vk --user ${OPENECOMP_NEXUS_USER}:${OPENECOMP_NEXUS_PASSWORD} --upload-file ${OUTPUT_DIR}/${OUTPUT_FILE_DATE_STAMPED} ${SEND_TO} +``` diff --git a/cdap3vm/config/cdap-config-template/cdap-env.sh b/cdap3vm/config/cdap-config-template/cdap-env.sh new file mode 100644 index 0000000..eb962c9 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-env.sh @@ -0,0 +1,26 @@ +# Copyright © 2015 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Example environment variables. Please refer to the CDAP Administration Manual +# for more information. + +# If running CDAP on HDP 2.2+ the full HDP version string including iteration +# number must be passed in as an option. + +export JAVA_HOME=__JAVA_HOME__ +export OPTS="${OPTS} -Dhdp.version=__HDP_VERSION__" + +PATH=$PATH:__NODEJS_BIN__ +# Ensure SPARK_HOME is set for Spark support. +# export SPARK_HOME="/usr/lib/spark" diff --git a/cdap3vm/config/cdap-config-template/cdap-site.xml b/cdap3vm/config/cdap-config-template/cdap-site.xml new file mode 100644 index 0000000..c0eabda --- /dev/null +++ b/cdap3vm/config/cdap-config-template/cdap-site.xml @@ -0,0 +1,61 @@ + + + + + + + zookeeper.quorum + __HDP_ZOOKEEPER_QUORUM__/${root.namespace} + + ZooKeeper quorum string; specifies the ZooKeeper host:port; + substitute the quorum for the components shown here (FQDN1:2181,FQDN2:2181) + + + + router.server.address + __CDAP_ROUTER_HOST__ + CDAP Router address to which CDAP UI connects + + + explore.enabled + false + Enable Explore functionality + + + enable.unrecoverable.reset + true + Needed for allowing deletion of namespaces + + + app.program.jvm.opts + -XX:MaxPermSize=128M ${twill.jvm.gc.opts} -Dhdp.version=__HDP_VERSION__ -Dspark.yarn.am.extraJavaOptions=-Dhdp.version=__HDP_VERSION__ + Java options for all program containers + + + + kafka.seed.brokers + __CDAP_KAFKA_HOST__:9092 + + Comma-separated list of CDAP Kafka service brokers; for distributed CDAP, + replace with list of FQDN:port brokers + + + + + diff --git a/cdap3vm/config/cdap-config-template/common/common.sh b/cdap3vm/config/cdap-config-template/common/common.sh new file mode 100755 index 0000000..c58fb6a --- /dev/null +++ b/cdap3vm/config/cdap-config-template/common/common.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash + +# +# Copyright © 2015-2016 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# checks if there exists a PID that is already running. return 0 idempotently + +export JAVA_HOME=__JAVA_HOME__ +PATH=$PATH:__NODEJS_BIN__ + +cdap_check_before_start() { + if [ -f ${pid} ]; then + if kill -0 $(<${pid}) > /dev/null 2>&1; then + echo "${APP} running as process $(<${pid}). Stop it first." + return 1 + fi + fi + return 0 +} + +cdap_create_pid_dir() { + mkdir -p "${PID_DIR}" +} + +die() { + echo "ERROR: ${*}" + exit 1 +} + +# usage: cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true +cdap_get_conf() { + local __pn=${1} __fn=${2} __default=${3} __result= + # Check for xmllint + [[ $(which xmllint 2>/dev/null) ]] || { + case ${PLATFORM} in + RHEL) die "Cannot locate xmllint, is libxml2 installed?" ;; + UBUNTU) die "Cannot locate xmllint, is libxml2-utils installed?" ;; + esac + # If we get here, die + die "Cannot locate xmllint, are XML tools installed?" + } + # Get property from file, return last result, if multiple are returned + __property="cat //configuration/property[name='${__pn}']/value[text()]" + __sed_fu='/^\//d;s/^.*//;s/<\/value>.*$//' + __result=$(echo "${__property}" | xmllint --shell "${__fn}" | sed "${__sed_fu}" | tail -n 1) + # Found result, echo it and return 0 + [[ -n "${__result}" ]] && echo ${__result} && return 0 + # No result, echo default and return 0 + [[ -n "${__default}" ]] && echo ${__default} && return 0 + return 1 +} + +# Rotates the basic start/stop logs +cdap_rotate_log () { + local log=${1} num=5 prev=0 + [[ -n "${2}" ]] && num=${2} + if [ -f "${log}" ]; then # rotate logs + while [ ${num} -gt 1 ]; do + prev=$((${num} - 1)) + [ -f "${log}.${prev}" ] && mv -f "${log}.${prev}" "${log}.${num}" + num=${prev} + done + mv -f "${log}" "${log}.${num}" + fi +} + +# CDAP kinit using properties from cdap-site.xml +cdap_kinit() { + local __principal=${CDAP_PRINCIPAL:-$(cdap_get_conf "cdap.master.kerberos.principal" "${CDAP_CONF}"/cdap-site.xml)} + local __keytab=${CDAP_KEYTAB:-$(cdap_get_conf "cdap.master.kerberos.keytab" "${CDAP_CONF}"/cdap-site.xml)} + if [ -z "${__principal}" -o -z "${__keytab}" ]; then + echo "ERROR: Both cdap.master.kerberos.principal and cdap.master.kerberos.keytab must be configured for Kerberos-enabled clusters!" + return 1 + fi + if [ ! -r "${__keytab}" ]; then + echo "ERROR: Cannot read keytab: ${__keytab}" + return 1 + fi + if [[ $(which kinit 2>/dev/null) ]]; then + # Replace _HOST in principal w/ FQDN, like Hadoop does + kinit -kt "${__keytab}" "${__principal/_HOST/`hostname -f`}" + if [[ ! $? ]]; then + echo "ERROR: Failed executing 'kinit -kt \"${__keytab}\" \"${__principal/_HOST/`hostname -f`}\"'" + return 1 + fi + else + echo "ERROR: Cannot locate kinit! Please, ensure the appropriate Kerberos utilities are installed" + return 1 + fi + return 0 +} + +# Attempts to find JAVA in few ways. This is used by UI's config-tool, so verify that changes here do not break that +cdap_set_java () { + # Determine the Java command to use to start the JVM. + if [ -n "${JAVA_HOME}" ] ; then + if [ -x "${JAVA_HOME}/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + export JAVA="${JAVA_HOME}/jre/sh/java" + else + export JAVA="${JAVA_HOME}/bin/java" + fi + if [ ! -x "${JAVA}" ] ; then + echo "ERROR: JAVA_HOME is set to an invalid directory: ${JAVA_HOME} + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + else + export JAVA="java" + which java >/dev/null 2>&1 && return 0 + # If we get here, we've failed this city + echo "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." >&2 + return 1 + fi + return 0 +} + +# Sets the correct HBase support library to use, based on what version exists in the classpath +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hbase() { + # Why is this here? Is this not redundant? + if [ -z "${JAVA}" ]; then + echo "ERROR: JAVA is not yet set, cannot determine HBase version" + return 1 + fi + + retvalue=1 + if [ -z "${HBASE_VERSION}" ]; then + HBASE_VERSION=$(${JAVA} -cp ${CLASSPATH} co.cask.cdap.data2.util.hbase.HBaseVersion 2> /dev/null) + retvalue=$? + else + retvalue=0 + fi + + # only set HBase version if previous call succeeded (may fail for components that don't use HBase) + if [ ${retvalue} == 0 ]; then + case ${HBASE_VERSION} in + 0.96*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.96/lib/*" + ;; + 0.98*) + hbasecompat="${CDAP_HOME}/hbase-compat-0.98/lib/*" + ;; + 1.0-cdh5.5* | 1.0-cdh5.6*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh5.5.0/lib/*" + ;; + 1.0-cdh*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0-cdh/lib/*" + ;; + 1.0*) + hbasecompat="${CDAP_HOME}/hbase-compat-1.0/lib/*" + ;; + 1.1*) + hbasecompat="$CDAP_HOME/hbase-compat-1.1/lib/*" + ;; + *) + echo "ERROR: Unknown/unsupported version of HBase found: ${HBASE_VERSION}" + return 1 + ;; + esac + if [ -n "${hbasecompat}" ]; then + CLASSPATH="${hbasecompat}:${CLASSPATH}" + else + # When will we ever hit this case? + echo "ERROR: Failed to find installed hbase-compat jar for version ${HBASE_VERSION}." + echo " Is the hbase-compat-* package installed?" + return 1 + fi + fi + export CLASSPATH + return 0 +} + +# set the classpath to include hadoop and hbase dependencies +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_classpath() { + COMP_HOME=${1} + CCONF=${2} + if [ -n "${HBASE_HOME}" ]; then + HBASE_CP=$(${HBASE_HOME}/bin/hbase classpath) + elif [[ $(which hbase 2>/dev/null) ]]; then + HBASE_CP=$(hbase classpath) + fi + + # Where is this used outside this function? + export HBASE_CP + + # In order to ensure that we can do hacks, need to make sure classpath is sorted + # so that cdap jars are placed earlier in the classpath than twill or hadoop jars + COMP_LIB=$(find -L "${COMP_HOME}/lib" -type f | sort | tr '\n' ':') + + if [ -n "${HBASE_CP}" ]; then + CP="${COMP_LIB}:${HBASE_CP}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + else + # assume Hadoop/HBase libs are included via EXTRA_CLASSPATH + echo "WARN: could not find Hadoop and HBase libraries" + CP="${COMP_LIB}:${CCONF}/:${COMP_HOME}/conf/:${EXTRA_CLASSPATH}" + fi + + # Setup classpaths. + if [ -n "${CLASSPATH}" ]; then + CLASSPATH="${CLASSPATH}:${CP}" + else + CLASSPATH="${CP}" + fi + + export CLASSPATH +} + +# Determine Hive classpath, and set EXPLORE_CLASSPATH. +# Hive classpath is not added as part of system classpath as hive jars bundle unrelated jars like guava, +# and hence need to be isolated. +# NOTE: this function is also sourced and invoked by the CSD control script, found here: +# https://github.com/caskdata/cm_csd/blob/develop/src/scripts/cdap-control.sh +# Any changes to this function must be compatible with the CSD's invocation +cdap_set_hive_classpath() { + local __explore=${EXPLORE_ENABLED:-$(cdap_get_conf "explore.enabled" "${CDAP_CONF}"/cdap-site.xml true)} + if [[ "${__explore}" == "true" ]]; then + if [ -z "${HIVE_HOME}" -o -z "${HIVE_CONF_DIR}" -o -z "${HADOOP_CONF_DIR}" ]; then + __secure=${KERBEROS_ENABLED:-$(cdap_get_conf "kerberos.auth.enabled" "${CDAP_CONF}"/cdap-site.xml false)} + if [[ "${__secure}" == "true" ]]; then + cdap_kinit || return 1 + fi + + if [[ $(which hive 2>/dev/null) ]]; then + ERR_FILE=$(mktemp) + HIVE_VAR_OUT=$(hive -e 'set -v' 2>${ERR_FILE}) + __ret=$? + HIVE_ERR_MSG=$(< ${ERR_FILE}) + rm ${ERR_FILE} + if [ ${__ret} -ne 0 ]; then + echo "ERROR - While determining Hive classpath, failed to get Hive settings using: hive -e 'set -v'" + echo "If you do not want run CDAP with Hive functionality, set the 'explore.enabled' property in cdap-site.xml to 'false'" + echo "Otherwise, check that the Hive client is installed, and that Hive and HDFS are running." + echo "stderr:" + echo "${HIVE_ERR_MSG}" + return 1 + fi + HIVE_VARS=$(echo ${HIVE_VAR_OUT} | tr ' ' '\n') + # Quotes preserve whitespace + HIVE_HOME=${HIVE_HOME:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_HOME=' | cut -d= -f2)} + HIVE_CONF_DIR=${HIVE_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HIVE_CONF_DIR=' | cut -d= -f2)} + HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-$(echo -e "${HIVE_VARS}" | grep '^env:HADOOP_CONF_DIR=' | cut -d= -f2)} + fi + fi + + # If Hive classpath is successfully determined, derive explore + # classpath from it and export it to use it in the launch command + if [ -n "${HIVE_HOME}" -a -n "${HIVE_CONF_DIR}" -a -n "${HADOOP_CONF_DIR}" ]; then + EXPLORE_CONF_FILES=$(ls -1dF ${HIVE_CONF_DIR}/* ${HADOOP_CONF_DIR}/* | sed -e '/\/$/d' | tr '\n' ':') + EXPLORE_CLASSPATH=$(ls -1 ${HIVE_HOME}/lib/hive-exec-* ${HIVE_HOME}/lib/*.jar | tr '\n' ':') + export EXPLORE_CONF_FILES EXPLORE_CLASSPATH + fi + fi +} + +# Check that directory /var/tmp/cdap exists in the master node, or create it +cdap_check_or_create_master_local_dir() { + mkdir -p "${LOCAL_DIR}" +} + +# check and set classpath if in development enviroment +cdap_check_and_set_classpath_for_dev_environment () { + APP_HOME=${1} + + # Detect if we are in development. + IN_DEV_ENVIRONMENT=${IN_DEV_ENVIRONMENT:-false} + + # for developers only, add flow and flow related stuff to class path. + if [[ "${IN_DEV_ENVIRONMENT}" == "true" ]]; then + echo "Constructing classpath for development environment ..." + [[ -f "${APP_HOME}"/build/generated-classpath ]] && CLASSPATH+=":$(<${APP_HOME}/build/generated-classpath)" + [[ -d "${APP_HOME}"/build/classes ]] && CLASSPATH+=":${APP_HOME}/build/classes/main:${APP_HOME}/conf/*" + [[ -d "${APP_HOME}"/../data-fabric/build/classes ]] && CLASSPATH+=":${APP_HOME}/../data-fabric/build/classes/main" + [[ -d "${APP_HOME}"/../common/build/classes ]] && CLASSPATH+=":${APP_HOME}/../common/build/classes/main" + [[ -d "${APP_HOME}"/../gateway/build/classes ]] && CLASSPATH+=":${APP_HOME}/../gateway/build/classes/main" + export CLASSPATH + fi +} + +HOSTNAME=$(hostname -f) +export LOG_PREFIX=${APP}-${IDENT_STRING}-${HOSTNAME} +export LOGFILE=${LOG_PREFIX}.log +loglog="${LOG_DIR}/${LOGFILE}" + +pid=${PID_DIR}/${APP}-${IDENT_STRING}.pid +loggc="${LOG_DIR}/${LOG_PREFIX}.gc" + +export NICENESS=${NICENESS:-0} diff --git a/cdap3vm/config/cdap-config-template/logback-container.xml b/cdap3vm/config/cdap-config-template/logback-container.xml new file mode 100644 index 0000000..f1957d8 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback-container.xml @@ -0,0 +1,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${LOG_DIRS}/program.log + + %d{ISO8601} - %-5p [%t:%logger{1}@%L] - %m%n + + + + ${LOG_DIRS}/program.%d.log + + + 14 + + + + + + + + + diff --git a/cdap3vm/config/cdap-config-template/logback.xml b/cdap3vm/config/cdap-config-template/logback.xml new file mode 100644 index 0000000..e036683 --- /dev/null +++ b/cdap3vm/config/cdap-config-template/logback.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + + + + + + + + + + + %d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + + + + + + + + + + + diff --git a/cdap3vm/config/hadoop-cluster-conf-file.sh b/cdap3vm/config/hadoop-cluster-conf-file.sh new file mode 100644 index 0000000..a5fcf3b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-conf-file.sh @@ -0,0 +1,131 @@ +#!/bin/sh + +# +# Based on HortonWorks Directories Script +# + +# __HDP_VERSION__=2.3.2.0-2950 +__HDP_VERSION__=2.4.3.0-227 +# Space separated list of directories where NameNode will store file system image. +# For example, /grid/hadoop/hdfs/nn /grid1/hadoop/hdfs/nn +__DFS_NAME_DIR__=/opt/data/hadoop/hdfs/namenode + +# Space separated list of directories where DataNodes will store the blocks. +# For example, /grid/hadoop/hdfs/dn /grid1/hadoop/hdfs/dn /grid2/hadoop/hdfs/dn +__DFS_DATA_DIR__=/opt/data/hadoop/hdfs/data + +# Space separated list of directories where SecondaryNameNode will store checkpoint image. +# For example, /grid/hadoop/hdfs/snn /grid1/hadoop/hdfs/snn /grid2/hadoop/hdfs/snn +__FS_CHECKPOINT_DIR__=/opt/data/hadoop/hdfs/snamenode + +# Directory to store the HDFS logs. +__HDFS_LOG_DIR__=/opt/data/log/hadoop/hdfs + +# Directory to store the HDFS process ID. +__HDFS_PID_DIR__=/var/run/hadoop/hdfs + +# Directory to store the Hadoop configuration files. +__HADOOP_CONF_DIR__=/etc/hadoop/conf + +# Main Hadoop LOG dir +__HADOOP_LOG_MAIN_DIR__=/opt/data/log + +# +# Hadoop Service - YARN +# + +# Space separated list of directories where YARN will store temporary data. +# For example, /grid/hadoop/yarn/local /grid1/hadoop/yarn/local /grid2/hadoop/yarn/local +__YARN_LOCAL_DIR__=/opt/data/hadoop/yarn/local + +# Directory to store the YARN logs. +__YARN_LOG_DIR__=/opt/data/log/hadoop/yarn + +# Directory for nodemanager.recovery +__YARN_NODEMANAGER_RECOVERY_DIR__=/opt/data/log/hadoop-yarn/nodemanager/recovery-state + + +# Space separated list of directories where YARN will store container log data. +# For example, /grid/hadoop/yarn/logs /grid1/hadoop/yarn/logs /grid2/hadoop/yarn/logs +__YARN_LOCAL_LOG_DIR__=/opt/data/hadoop/yarn/log + +# Directory to store the YARN process ID. +__YARN_PID_DIR__=/var/run/hadoop/yarn + +# +# Hadoop Service - MAPREDUCE +# + +# Directory to store the MapReduce daemon logs. +__MAPRED_LOG_DIR__=/opt/data/log/hadoop/mapred + +# Directory to store the mapreduce jobhistory process ID. +__MAPRED_PID_DIR__=/var/run/hadoop/mapred + +# +# Hadoop Service - HBase +# + +# Directory to store the HBase configuration files. +__HBASE_CONF_DIR__=/etc/hbase/conf + +# Directory to store the HBase logs. +__HBASE_LOG_DIR__=/opt/data/log/hbase + +# Directory to store the HBase logs. +__HBASE_PID_DIR__=/var/run/hbase + +# +# Hadoop Service - ZooKeeper +# + +# Directory where ZooKeeper will store data. For example, /grid1/hadoop/zookeeper/data +__ZOOKEEPER_DATA_DIR__=/opt/data/hadoop/zookeeper/data + +# Directory to store the ZooKeeper configuration files. +__ZOOKEEPER_CONF_DIR__=/etc/zookeeper/conf + +# Directory to store the ZooKeeper logs. +__ZOOKEEPER_LOG_DIR__=/opt/data/log/zookeeper +__ZKFC_LOG_DIR__=/opt/data/log/hdfs + +# Directory to store the ZooKeeper process ID. +__ZOOKEEPER_PID_DIR__=/var/run/zookeeper + + +# +# User Accounts +# + +__SUPER_USER__=root +__HADOOP_USER__=hadoop +__HDFS_USER__=hdfs +__YARN_USER__=yarn +__HBASE_USER__=hbase +__CDAP_USER__=cdap +__ZOOKEEPER_USER__=zookeeper +__MAPRED_USER__=mapred +__HADOOP_GROUP__=hadoop + +# +# Other service folder locations +# +__SERVICE_CONFIG_FOLDER__=/etc/hadoop/service_scripts +__HDP_CURRENT_FOLDER__=/usr/hdp/current +__HDP_HADOOP_BIN__=/usr/hdp/current/hadoop-client/bin +__HDP_HADOOP_SBIN__=/usr/hdp/current/hadoop-client/sbin +__RPM_FOLDER_LOCATION__=pkgs + +# __RPM_FOLDER_LOCATION__ is relative to main package directory +__HDP_RPM_ROOT__=UNUSED +__CDAP_RPM_ROOT__=UNUSED +__CENTOS7_RPM_ROOT__=UNUSED +# +# Main System +# +__JAVA_HOME__=/opt/app/java/jdk/jdk170 +__CDAP_CONF_DIR__=/etc/cdap/conf +__CDAP_INST_FOLDER__=/opt/cdap +__NODEJS_BIN__=/opt/app/nodejs/bin + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml new file mode 100644 index 0000000..0ab575e --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/capacity-scheduler.xml @@ -0,0 +1,79 @@ + + + + + yarn.scheduler.capacity.default.minimum-user-limit-percent + 100 + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.2 + + + + yarn.scheduler.capacity.maximum-applications + 10000 + + + + yarn.scheduler.capacity.node-locality-delay + 40 + + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator + + + + yarn.scheduler.capacity.root.accessible-node-labels + * + + + + yarn.scheduler.capacity.root.acl_administer_queue + * + + + + yarn.scheduler.capacity.root.capacity + 100 + + + + yarn.scheduler.capacity.root.default.acl_administer_jobs + * + + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + + + yarn.scheduler.capacity.root.default.capacity + 100 + + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + + + yarn.scheduler.capacity.root.default.state + RUNNING + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + + + yarn.scheduler.capacity.root.queues + default + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg new file mode 100644 index 0000000..d009d0d --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/container-executor.cfg @@ -0,0 +1,23 @@ + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +yarn.nodemanager.local-dirs=/grid/0/hadoop/yarn/local +yarn.nodemanager.log-dirs=/grid/0/hadoop/yarn/log +yarn.nodemanager.linux-container-executor.group=hadoop +banned.users=hdfs,yarn,mapred,bin +min.user.id=500 diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml new file mode 100644 index 0000000..83bfc2c --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/core-site.xml @@ -0,0 +1,147 @@ + + + + + fs.defaultFS + hdfs://__HDP_NAMENODE__:8020 + true + + + + fs.trash.interval + 360 + + + + ha.failover-controller.active-standby-elector.zk.op.retries + 120 + + + + hadoop.http.authentication.simple.anonymous.allowed + true + + + + hadoop.proxyuser.falcon.groups + users + + + + hadoop.proxyuser.falcon.hosts + * + + + + hadoop.proxyuser.hcat.groups + users + + + + hadoop.proxyuser.hcat.hosts + __HDP_WEBHCAT_SERVER__ + + + + hadoop.proxyuser.hdfs.groups + * + + + + hadoop.proxyuser.hdfs.hosts + * + + + + hadoop.proxyuser.hive.groups + users + + + + hadoop.proxyuser.hive.hosts + __HDP_HIVE_SERVER__ + + + + hadoop.proxyuser.oozie.groups + * + + + + hadoop.proxyuser.oozie.hosts + __HDP_OOZIE_SERVER__ + + + + hadoop.security.auth_to_local + DEFAULT + + + + hadoop.security.authentication + simple + + + + hadoop.security.authorization + false + + + + hadoop.security.key.provider.path + + + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec + + + + io.file.buffer.size + 131072 + + + + io.serializations + org.apache.hadoop.io.serializer.WritableSerialization + + + + ipc.client.connect.max.retries + 50 + + + + ipc.client.connection.maxidletime + 30000 + + + + ipc.client.idlethreshold + 8000 + + + + ipc.server.tcpnodelay + true + + + + mapreduce.jobtracker.webinterface.trusted + false + + + + + + proxyuser_group + users + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh new file mode 100644 index 0000000..f4a200c --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-env.sh @@ -0,0 +1,125 @@ + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +export JAVA_HOME=__JAVA_HOME__ +export HADOOP_HOME_WARN_SUPPRESS=1 + +export HADOOP_LOG_MAIN=__HADOOP_LOG_MAIN_DIR__ +# Hadoop home directory +export HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} + +# Hadoop Configuration Directory + + +# Path to jsvc required by secure HDP 2.0 datanode +export JSVC_HOME=/usr/lib/bigtop-utils + + +# The maximum amount of heap to use, in MB. Default is 1000. +export HADOOP_HEAPSIZE="1024" + +export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms1024m" + +# Extra Java runtime options. Empty by default. +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" + +# Command specific options appended to HADOOP_OPTS when specified +HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" + +HADOOP_TASKTRACKER_OPTS="-server -Xmx1024m -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" + + +SHARED_HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=128m -XX:MaxNewSize=128m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT" +export HADOOP_NAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}" +export HADOOP_DATANODE_OPTS="-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HADOOP_LOG_MAIN}/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:${HADOOP_LOG_MAIN}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms1024m -Xmx1024m -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}" + +export HADOOP_SECONDARYNAMENODE_OPTS="${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\" ${HADOOP_SECONDARYNAMENODE_OPTS}" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" + + +HADOOP_NFS3_OPTS="-Xmx1024m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}" +HADOOP_BALANCER_OPTS="-server -Xmx1024m ${HADOOP_BALANCER_OPTS}" + + +# On secure datanodes, user to run the datanode as after dropping privileges +export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-""} + +# Extra ssh options. Empty by default. +export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" + +# Where log files are stored. $HADOOP_HOME/logs by default. +export HADOOP_LOG_DIR=${HADOOP_LOG_MAIN}/$USER + +# History server logs +export HADOOP_MAPRED_LOG_DIR=${HADOOP_LOG_MAIN}-mapreduce/$USER + +# Where log files are stored in the secure data environment. +export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_MAIN}/$HADOOP_SECURE_DN_USER + +# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. +# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + +# host:path where hadoop code should be rsync'd from. Unset by default. +# export HADOOP_MASTER=master:/home/$USER/src/hadoop + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HADOOP_SLAVE_SLEEP=0.1 + +# The directory where pid files are stored. /tmp by default. +export HADOOP_PID_DIR=/var/run/hadoop/$USER +export HADOOP_SECURE_DN_PID_DIR=/var/run/hadoop/$HADOOP_SECURE_DN_USER + +# History server pid +export HADOOP_MAPRED_PID_DIR=/var/run/hadoop-mapreduce/$USER + +YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. + +# export HADOOP_NICENESS=10 + +# Use libraries from standard classpath +JAVA_JDBC_LIBS="" +#Add libraries required by mysql connector +for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +# Add libraries required by oracle connector +for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +# Add libraries required by nodemanager +MAPREDUCE_LIBS=/usr/hdp/current/hadoop-mapreduce-client/* +export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} + +# added to the HADOOP_CLASSPATH +if [ -d "/usr/hdp/current/tez-client" ]; then + if [ -d "/usr/hdp/current/tez-client/conf/" ]; then + # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP. + export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/ + fi +fi + + +# Setting path to hdfs command line +export HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec + +# Mostly required for hadoop 2.0 +export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH} + +export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml new file mode 100644 index 0000000..f6e8acf --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hadoop-policy.xml @@ -0,0 +1,59 @@ + + + + + security.admin.operations.protocol.acl + hadoop + + + + security.client.datanode.protocol.acl + * + + + + security.client.protocol.acl + * + + + + security.datanode.protocol.acl + * + + + + security.inter.datanode.protocol.acl + * + + + + security.inter.tracker.protocol.acl + * + + + + security.job.client.protocol.acl + * + + + + security.job.task.protocol.acl + * + + + + security.namenode.protocol.acl + * + + + + security.refresh.policy.protocol.acl + hadoop + + + + security.refresh.usertogroups.mappings.protocol.acl + hadoop + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml new file mode 100644 index 0000000..a9c9c0f --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/hdfs-site.xml @@ -0,0 +1,281 @@ + + + + + dfs.block.access.token.enable + true + + + + dfs.blockreport.initialDelay + 120 + + + + dfs.blocksize + 134217728 + + + + dfs.client.read.shortcircuit + true + + + + dfs.client.read.shortcircuit.streams.cache.size + 4096 + + + + dfs.client.retry.policy.enabled + false + + + + dfs.cluster.administrators + hdfs + + + + dfs.datanode.address + 0.0.0.0:50010 + + + + dfs.datanode.balance.bandwidthPerSec + 6250000 + + + + dfs.datanode.data.dir + __DFS_DATA_DIR__ + + + + dfs.datanode.data.dir.perm + 750 + + + + dfs.datanode.du.reserved + 1073741824 + + + + dfs.datanode.failed.volumes.tolerated + 0 + + + + dfs.datanode.http.address + 0.0.0.0:50075 + + + + dfs.datanode.https.address + 0.0.0.0:50475 + + + + dfs.datanode.ipc.address + 0.0.0.0:8010 + + + + dfs.datanode.max.transfer.threads + 16384 + + + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + + + dfs.encryption.key.provider.uri + + + + + dfs.heartbeat.interval + 3 + + + + dfs.hosts.exclude + /etc/hadoop/conf/dfs.exclude + + + + dfs.http.policy + HTTP_ONLY + + + + dfs.https.port + 50470 + + + + dfs.journalnode.edits.dir + /hadoop/hdfs/journalnode + + + + dfs.journalnode.http-address + 0.0.0.0:8480 + + + + dfs.journalnode.https-address + 0.0.0.0:8481 + + + + dfs.namenode.accesstime.precision + 0 + + + + dfs.namenode.audit.log.async + true + + + + dfs.namenode.avoid.read.stale.datanode + true + + + + dfs.namenode.avoid.write.stale.datanode + true + + + + dfs.namenode.checkpoint.dir + __FS_CHECKPOINT_DIR__ + + + + dfs.namenode.checkpoint.edits.dir + ${dfs.namenode.checkpoint.dir} + + + + dfs.namenode.checkpoint.period + 21600 + + + + dfs.namenode.checkpoint.txns + 1000000 + + + + dfs.namenode.fslock.fair + false + + + + dfs.namenode.handler.count + 200 + + + + dfs.namenode.http-address + __HDP_NAMENODE__:50070 + true + + + + dfs.namenode.https-address + __HDP_NAMENODE__:50470 + + + + dfs.namenode.name.dir + __DFS_NAME_DIR__ + + + + dfs.namenode.name.dir.restore + true + + + + dfs.namenode.rpc-address + __HDP_NAMENODE__:8020 + + + + dfs.namenode.safemode.threshold-pct + 1 + + + + dfs.namenode.secondary.http-address + __HDP_SECONDARY_NAMENODE__:50090 + + + + dfs.namenode.stale.datanode.interval + 30000 + + + + dfs.namenode.startup.delay.block.deletion.sec + 3600 + + + + dfs.namenode.write.stale.datanode.ratio + 1.0f + + + + dfs.permissions.enabled + true + + + + dfs.permissions.superusergroup + hdfs + + + + dfs.replication + 3 + + + + dfs.replication.max + 50 + + + + dfs.support.append + true + true + + + + dfs.webhdfs.enabled + false + + + + fs.permissions.umask-mode + 022 + + + + nfs.exports.allowed.hosts + * rw + + + + nfs.file.dump.dir + /tmp/.hdfs-nfs + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check new file mode 100644 index 0000000..909d249 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/health_check @@ -0,0 +1,64 @@ + +#!/bin/bash +# +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +err=0; + +function check_disks { + + for m in `awk '$3~/ext3/ {printf" %s ",$2}' /etc/fstab` ; do + fsdev="" + fsdev=`awk -v m=$m '$2==m {print $1}' /proc/mounts`; + if [ -z "$fsdev" -a "$m" != "/mnt" ] ; then + msg_="$msg_ $m(u)" + else + msg_="$msg_`awk -v m=$m '$2==m { if ( $4 ~ /^ro,/ ) {printf"%s(ro)",$2 } ; }' /proc/mounts`" + fi + done + + if [ -z "$msg_" ] ; then + echo "disks ok" ; exit 0 + else + echo "$msg_" ; exit 2 + fi + +} + +# Run all checks +for check in disks ; do + msg=`check_${check}` ; + if [ $? -eq 0 ] ; then + ok_msg="$ok_msg$msg," + else + err_msg="$err_msg$msg," + fi +done + +if [ ! -z "$err_msg" ] ; then + echo -n "ERROR $err_msg " +fi +if [ ! -z "$ok_msg" ] ; then + echo -n "OK: $ok_msg" +fi + +echo + +# Success! +exit 0 diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml new file mode 100644 index 0000000..cba69f4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-acls.xml @@ -0,0 +1,135 @@ + + + + + + + + + + hadoop.kms.acl.CREATE + * + + ACL for create-key operations. + If the user is not in the GET ACL, the key material is not returned + as part of the response. + + + + + hadoop.kms.acl.DELETE + * + + ACL for delete-key operations. + + + + + hadoop.kms.acl.ROLLOVER + * + + ACL for rollover-key operations. + If the user is not in the GET ACL, the key material is not returned + as part of the response. + + + + + hadoop.kms.acl.GET + * + + ACL for get-key-version and get-current-key operations. + + + + + hadoop.kms.acl.GET_KEYS + * + + ACL for get-keys operations. + + + + + hadoop.kms.acl.GET_METADATA + * + + ACL for get-key-metadata and get-keys-metadata operations. + + + + + hadoop.kms.acl.SET_KEY_MATERIAL + * + + Complementary ACL for CREATE and ROLLOVER operations to allow the client + to provide the key material when creating or rolling a key. + + + + + hadoop.kms.acl.GENERATE_EEK + * + + ACL for generateEncryptedKey CryptoExtension operations. + + + + + hadoop.kms.acl.DECRYPT_EEK + * + + ACL for decryptEncryptedKey CryptoExtension operations. + + + + + default.key.acl.MANAGEMENT + * + + default ACL for MANAGEMENT operations for all key acls that are not + explicitly defined. + + + + + default.key.acl.GENERATE_EEK + * + + default ACL for GENERATE_EEK operations for all key acls that are not + explicitly defined. + + + + + default.key.acl.DECRYPT_EEK + * + + default ACL for DECRYPT_EEK operations for all key acls that are not + explicitly defined. + + + + + default.key.acl.READ + * + + default ACL for READ operations for all key acls that are not + explicitly defined. + + + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh new file mode 100644 index 0000000..44dfe6a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-env.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# Set kms specific environment variables here. + +# Settings for the Embedded Tomcat that runs KMS +# Java System properties for KMS should be specified in this variable +# +# export CATALINA_OPTS= + +# KMS logs directory +# +# export KMS_LOG=${KMS_HOME}/logs + +# KMS temporary directory +# +# export KMS_TEMP=${KMS_HOME}/temp + +# The HTTP port used by KMS +# +# export KMS_HTTP_PORT=16000 + +# The Admin port used by KMS +# +# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1` + +# The maximum number of Tomcat handler threads +# +# export KMS_MAX_THREADS=1000 + +# The location of the SSL keystore if using SSL +# +# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore + +# The password of the SSL keystore if using SSL +# +# export KMS_SSL_KEYSTORE_PASS=password + +# The full path to any native libraries that need to be loaded +# (For eg. location of natively compiled tomcat Apache portable +# runtime (APR) libraries +# +# export JAVA_LIBRARY_PATH=${HOME}/lib/native diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml new file mode 100644 index 0000000..a810ca4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/kms-site.xml @@ -0,0 +1,173 @@ + + + + + + + + hadoop.kms.key.provider.uri + jceks://file@/${user.home}/kms.keystore + + URI of the backing KeyProvider for the KMS. + + + + + hadoop.security.keystore.JavaKeyStoreProvider.password + none + + If using the JavaKeyStoreProvider, the password for the keystore file. + + + + + + + hadoop.kms.cache.enable + true + + Whether the KMS will act as a cache for the backing KeyProvider. + When the cache is enabled, operations like getKeyVersion, getMetadata, + and getCurrentKey will sometimes return cached data without consulting + the backing KeyProvider. Cached values are flushed when keys are deleted + or modified. + + + + + hadoop.kms.cache.timeout.ms + 600000 + + Expiry time for the KMS key version and key metadata cache, in + milliseconds. This affects getKeyVersion and getMetadata. + + + + + hadoop.kms.current.key.cache.timeout.ms + 30000 + + Expiry time for the KMS current key cache, in milliseconds. This + affects getCurrentKey operations. + + + + + + + hadoop.kms.audit.aggregation.window.ms + 10000 + + Duplicate audit log events within the aggregation window (specified in + ms) are quashed to reduce log traffic. A single message for aggregated + events is printed at the end of the window, along with a count of the + number of aggregated events. + + + + + + + hadoop.kms.authentication.type + simple + + Authentication type for the KMS. Can be either "simple" + or "kerberos". + + + + + hadoop.kms.authentication.kerberos.keytab + ${user.home}/kms.keytab + + Path to the keytab with credentials for the configured Kerberos principal. + + + + + hadoop.kms.authentication.kerberos.principal + HTTP/localhost + + The Kerberos principal to use for the HTTP endpoint. + The principal must start with 'HTTP/' as per the Kerberos HTTP SPNEGO specification. + + + + + hadoop.kms.authentication.kerberos.name.rules + DEFAULT + + Rules used to resolve Kerberos principal names. + + + + + + + hadoop.kms.authentication.signer.secret.provider + random + + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. + + + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.path + /hadoop-kms/hadoop-auth-signature-secret + + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string + #HOSTNAME#:#PORT#,... + + The Zookeeper connection string, a list of hostnames and port comma + separated. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type + kerberos + + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab + /etc/hadoop/conf/kms.keytab + + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. + + + + + hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal + kms/#HOSTNAME# + + The Kerberos service principal used to connect to Zookeeper. + + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties new file mode 100644 index 0000000..9ad03cd --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/log4j.properties @@ -0,0 +1,225 @@ + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +# Define some default values that can be overridden by system properties +# To change daemon root logger use hadoop_root_logger in hadoop-env +hadoop.root.logger=INFO,console +hadoop.log.dir=. +hadoop.log.file=hadoop.log + + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hadoop.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=ALL + +# +# Daily Rolling File Appender +# + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# +# TaskLog Appender +# + +#Default values +hadoop.tasklog.taskid=null +hadoop.tasklog.iscleanup=false +hadoop.tasklog.noKeepSplits=4 +hadoop.tasklog.totalLogFileSize=100 +hadoop.tasklog.purgeLogSplits=true +hadoop.tasklog.logsRetainHours=12 + +log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender +log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} +log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} +log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} + +log4j.appender.TLA.layout=org.apache.log4j.PatternLayout +log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +#Security audit appender +# +hadoop.security.logger=INFO,console +hadoop.security.log.maxfilesize=256MB +hadoop.security.log.maxbackupindex=20 +log4j.category.SecurityLogger=${hadoop.security.logger} +hadoop.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd + +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} + +# +# hdfs audit logging +# +hdfs.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} +log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false +log4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log +log4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd + +# +# mapred audit logging +# +mapred.audit.logger=INFO,console +log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} +log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false +log4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender +log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log +log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout +log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd + +# +# Rolling File Appender +# + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} + +# Logfile size and and 30-day backups +log4j.appender.RFA.MaxFileSize=256MB +log4j.appender.RFA.MaxBackupIndex=10 + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# Custom Logging levels + +hadoop.metrics.log.level=INFO +#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG +#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level} + +# Jets3t library +log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR + +# +# Null Appender +# Trap security logger on the hadoop client side +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter + +# Removes "deprecated" messages +log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN + +# +# HDFS block state change log from block manager +# +# Uncomment the following to suppress normal block state change +# messages from BlockManager in NameNode. +#log4j.logger.BlockStateChange=WARN +#Relative to Yarn Log Dir Prefix +yarn.log.dir=. +# +# Job Summary Appender +# +# Use following logger to send summary to separate file defined by +# hadoop.mapreduce.jobsummary.log.file rolled daily: +# hadoop.mapreduce.jobsummary.logger=INFO,JSA +# +hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} +hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log +log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender +# Set the ResourceManager summary log filename +yarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log +# Set the ResourceManager summary log level and appender +yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} +#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY + +# To enable AppSummaryLogging for the RM, +# set yarn.server.resourcemanager.appsummary.logger to +# LEVEL,RMSUMMARY in hadoop-env.sh + +# Appender for ResourceManager Application Summary Log +# Requires the following properties to be set +# - hadoop.log.dir (Hadoop Log directory) +# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) +# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) +log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender +log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} +log4j.appender.RMSUMMARY.MaxFileSize=256MB +log4j.appender.RMSUMMARY.MaxBackupIndex=20 +log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout +log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n +log4j.appender.JSA.DatePattern=.yyyy-MM-dd +log4j.appender.JSA.layout=org.apache.log4j.PatternLayout +log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} +log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false + +# Appender for viewing information for errors and warnings +yarn.ewma.cleanupInterval=300 +yarn.ewma.messageAgeLimitSeconds=86400 +yarn.ewma.maxUniqueMessages=250 +log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender +log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} +log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} +log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh new file mode 100644 index 0000000..582cdd4 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-env.sh @@ -0,0 +1,14 @@ + +export JAVA_HOME=__JAVA_HOME__ + +export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=900 + +export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA + +#export HADOOP_JOB_HISTORYSERVER_OPTS= +#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default. +#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger. +#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default. +#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default +#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0. +export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml new file mode 100644 index 0000000..58b9ae1 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/mapred-site.xml @@ -0,0 +1,224 @@ + + + + + mapreduce.admin.map.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} + + + + mapreduce.admin.reduce.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} + + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64 + + + + mapreduce.am.max-attempts + 2 + + + + mapreduce.application.classpath + $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure + + + + mapreduce.application.framework.path + /hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework + + + + mapreduce.cluster.administrators + hadoop + + + + mapreduce.framework.name + yarn + + + + mapreduce.job.counters.max + 130 + + + + mapreduce.job.emit-timeline-data + false + + + + mapreduce.job.reduce.slowstart.completedmaps + 0.05 + + + + mapreduce.jobhistory.address + __HDP_HISTORYSERVER__:10020 + + + + mapreduce.jobhistory.bind-host + 0.0.0.0 + + + + mapreduce.jobhistory.done-dir + /mr-history/done + + + + mapreduce.jobhistory.intermediate-done-dir + /mr-history/tmp + + + + mapreduce.jobhistory.webapp.address + __HDP_HISTORYSERVER__:19888 + + + + mapreduce.map.java.opts + -Xmx1024m + + + + mapreduce.map.log.level + INFO + + + + mapreduce.map.memory.mb + 2560 + + + + mapreduce.map.output.compress + false + + + + mapreduce.map.sort.spill.percent + 0.7 + + + + mapreduce.map.speculative + false + + + + mapreduce.output.fileoutputformat.compress + false + + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + + mapreduce.reduce.input.buffer.percent + 0.0 + + + + mapreduce.reduce.java.opts + -Xmx4096m + + + + mapreduce.reduce.log.level + INFO + + + + mapreduce.reduce.memory.mb + 5120 + + + + mapreduce.reduce.shuffle.fetch.retry.enabled + 1 + + + + mapreduce.reduce.shuffle.fetch.retry.interval-ms + 1000 + + + + mapreduce.reduce.shuffle.fetch.retry.timeout-ms + 30000 + + + + mapreduce.reduce.shuffle.input.buffer.percent + 0.7 + + + + mapreduce.reduce.shuffle.merge.percent + 0.66 + + + + mapreduce.reduce.shuffle.parallelcopies + 30 + + + + mapreduce.reduce.speculative + false + + + + mapreduce.shuffle.port + 13562 + + + + mapreduce.task.io.sort.factor + 100 + + + + mapreduce.task.io.sort.mb + 1433 + + + + mapreduce.task.timeout + 300000 + + + + yarn.app.mapreduce.am.admin-command-opts + -Dhdp.version=${hdp.version} + + + + yarn.app.mapreduce.am.command-opts + -Xmx2048m -Dhdp.version=${hdp.version} + + + + yarn.app.mapreduce.am.log.level + INFO + + + + yarn.app.mapreduce.am.resource.mb + 2560 + + + + yarn.app.mapreduce.am.staging-dir + /user + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml new file mode 100644 index 0000000..df08d8e --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-client.xml @@ -0,0 +1,39 @@ + + + + + ssl.client.keystore.location + /etc/security/clientKeys/keystore.jks + + + + ssl.client.keystore.password + bigdata + + + + ssl.client.keystore.type + jks + + + + ssl.client.truststore.location + /etc/security/clientKeys/all.jks + + + + ssl.client.truststore.password + bigdata + + + + ssl.client.truststore.reload.interval + 10000 + + + + ssl.client.truststore.type + jks + + + \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml new file mode 100644 index 0000000..3ba6399 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/ssl-server.xml @@ -0,0 +1,44 @@ + + + + + ssl.server.keystore.keypassword + bigdata + + + + ssl.server.keystore.location + /etc/security/serverKeys/keystore.jks + + + + ssl.server.keystore.password + bigdata + + + + ssl.server.keystore.type + jks + + + + ssl.server.truststore.location + /etc/security/serverKeys/all.jks + + + + ssl.server.truststore.password + bigdata + + + + ssl.server.truststore.reload.interval + 10000 + + + + ssl.server.truststore.type + jks + + + \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg new file mode 100644 index 0000000..e0d5309 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/taskcontroller.cfg @@ -0,0 +1,21 @@ + +#/* +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +mapred.local.dir=/tmp/hadoop-mapred/mapred/local +mapreduce.tasktracker.group=hadoop +hadoop.log.dir=__MAPRED_LOG_DIR__ diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh new file mode 100755 index 0000000..0f858c9 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-env.sh @@ -0,0 +1,119 @@ + + export HADOOP_YARN_HOME=/usr/hdp/current/hadoop-yarn-nodemanager + export YARN_LOG_DIR=__YARN_LOG_DIR__/$USER + export YARN_PID_DIR=__YARN_PID_DIR__/$USER + export HADOOP_LIBEXEC_DIR=__HDP_CURRENT_FOLDER__/hadoop-client/libexec + export JAVA_HOME=__JAVA_HOME__ + + # We need to add the EWMA appender for the yarn daemons only; + # however, YARN_ROOT_LOGGER is shared by the yarn client and the + # daemons. This is restrict the EWMA appender to daemons only. + INVOKER="${0##*/}" + if [ "$INVOKER" == "yarn-daemon.sh" ]; then + export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA} + fi + + # User for YARN daemons + export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + + # resolve links - $0 may be a softlink + export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" + + # some Java parameters + # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME + fi + + if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 + fi + + JAVA=$JAVA_HOME/bin/java + JAVA_HEAP_MAX=-Xmx1000m + + # For setting YARN specific HEAP sizes please use this + # Parameter and set appropriately + YARN_HEAPSIZE=1024 + + # check envvars which might override default args + if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" + fi + + # Resource Manager specific parameters + + # Specify the max Heapsize for the ResourceManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_RESOURCEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_RESOURCEMANAGER_HEAPSIZE=1024 + + # Specify the JVM options to be used when starting the ResourceManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_RESOURCEMANAGER_OPTS= + + # Node Manager specific parameters + + # Specify the max Heapsize for the NodeManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_NODEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_NODEMANAGER_HEAPSIZE=1024 + + # Specify the max Heapsize for the HistoryManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1024. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_HISTORYSERVER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_HISTORYSERVER_HEAPSIZE=1024 + + # Specify the JVM options to be used when starting the NodeManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_NODEMANAGER_OPTS= + + # so that filenames w/ spaces are handled correctly in loops below + IFS= + + + # default log directory and file + if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" + fi + if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' + fi + + # default policy file for service-level authorization + if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" + fi + + # restore ordinary behaviour + unset IFS + + + YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" + YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" + YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + fi + YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml new file mode 100644 index 0000000..bd96fb0 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn-site.xml @@ -0,0 +1,527 @@ + + + + + hadoop.registry.rm.enabled + true + + + + hadoop.registry.zk.quorum + __HDP_ZOOKEEPER_HOSTS_WITH_PORT__ + + + + yarn.acl.enable + false + + + + yarn.admin.acl + yarn + + + + yarn.application.classpath + __HADOOP_CONF_DIR__,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/* + + + + yarn.client.nodemanager-connect.max-wait-ms + 60000 + + + + yarn.client.nodemanager-connect.retry-interval-ms + 10000 + + + + yarn.http.policy + HTTP_ONLY + + + + yarn.log-aggregation-enable + true + + + + yarn.log-aggregation.retain-seconds + 2592000 + + + + yarn.log.server.url + http://__HDP_HISTORYSERVER__:19888/jobhistory/logs + + + + yarn.node-labels.enabled + false + + + + yarn.node-labels.fs-store.retry-policy-spec + 2000, 500 + + + + yarn.node-labels.fs-store.root-dir + /system/yarn/node-labels + + + + yarn.nodemanager.address + 0.0.0.0:45454 + + + + yarn.nodemanager.admin-env + MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + yarn.nodemanager.bind-host + 0.0.0.0 + + + + yarn.nodemanager.container-executor.class + org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor + + + + yarn.nodemanager.container-monitor.interval-ms + 3000 + + + + yarn.nodemanager.delete.debug-delay-sec + 43200 + + + + + yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage + 90 + + + + yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb + 1000 + + + + yarn.nodemanager.disk-health-checker.min-healthy-disks + 0.25 + + + + yarn.nodemanager.health-checker.interval-ms + 135000 + + + + yarn.nodemanager.health-checker.script.timeout-ms + 60000 + + + + yarn.nodemanager.linux-container-executor.cgroups.hierarchy + hadoop-yarn + + + + yarn.nodemanager.linux-container-executor.cgroups.mount + false + + + + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage + false + + + + yarn.nodemanager.linux-container-executor.group + hadoop + + + + yarn.nodemanager.linux-container-executor.resources-handler.class + org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler + + + + yarn.nodemanager.local-dirs + __YARN_LOCAL_DIR__ + + + + yarn.nodemanager.log-aggregation.compression-type + gz + + + + yarn.nodemanager.log-aggregation.debug-enabled + false + + + + yarn.nodemanager.log-aggregation.num-log-files-per-app + 30 + + + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds + -1 + + + + yarn.nodemanager.log-dirs + __YARN_LOG_DIR__ + + + + yarn.nodemanager.log.retain-second + 604800 + + + + yarn.nodemanager.recovery.dir + __YARN_NODEMANAGER_RECOVERY_DIR__ + + + + yarn.nodemanager.recovery.enabled + true + + + + yarn.nodemanager.remote-app-log-dir + /app-logs + + + + yarn.nodemanager.remote-app-log-dir-suffix + logs + + + + yarn.nodemanager.resource.cpu-vcores + 6 + + + + yarn.nodemanager.resource.memory-mb + 15360 + + + + yarn.nodemanager.resource.percentage-physical-cpu-limit + 80 + + + + yarn.nodemanager.vmem-check-enabled + false + + + + yarn.nodemanager.vmem-pmem-ratio + 2.1 + + + + yarn.resourcemanager.address + __HDP_RESOURCEMANAGER__:8050 + + + + yarn.resourcemanager.admin.address + __HDP_RESOURCEMANAGER__:8141 + + + + yarn.resourcemanager.am.max-attempts + 2 + + + + yarn.resourcemanager.bind-host + 0.0.0.0 + + + + yarn.resourcemanager.connect.max-wait.ms + 900000 + + + + yarn.resourcemanager.connect.retry-interval.ms + 30000 + + + + yarn.resourcemanager.fs.state-store.retry-policy-spec + 2000, 500 + + + + yarn.resourcemanager.fs.state-store.uri + + + + + yarn.resourcemanager.ha.enabled + false + + + + yarn.resourcemanager.hostname + __HDP_RESOURCEMANAGER__ + + + + yarn.resourcemanager.nodes.exclude-path + /etc/hadoop/conf/yarn.exclude + + + + yarn.resourcemanager.recovery.enabled + true + + + + yarn.resourcemanager.resource-tracker.address + __HDP_RESOURCEMANAGER__:8025 + + + + yarn.resourcemanager.scheduler.address + __HDP_RESOURCEMANAGER__:8030 + + + + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + + yarn.resourcemanager.scheduler.monitor.enable + false + + + + yarn.resourcemanager.state-store.max-completed-applications + ${yarn.resourcemanager.max-completed-applications} + + + + yarn.resourcemanager.store.class + org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore + + + + yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size + 10 + + + + yarn.resourcemanager.system-metrics-publisher.enabled + true + + + + yarn.resourcemanager.webapp.address + __HDP_RESOURCEMANAGER__:8088 + + + + yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled + false + + + + yarn.resourcemanager.webapp.https.address + __HDP_RESOURCEMANAGER__:8090 + + + + yarn.resourcemanager.work-preserving-recovery.enabled + true + + + + yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms + 10000 + + + + yarn.resourcemanager.zk-acl + world:anyone:rwcda + + + + yarn.resourcemanager.zk-address + __HDP_ZOOKEEPER_HOSTS_WITH_PORT__ + + + + yarn.resourcemanager.zk-num-retries + 1000 + + + + yarn.resourcemanager.zk-retry-interval-ms + 1000 + + + + yarn.resourcemanager.zk-state-store.parent-path + /rmstore + + + + yarn.resourcemanager.zk-timeout-ms + 10000 + + + + yarn.scheduler.maximum-allocation-mb + 8192 + + + + + yarn.scheduler.maximum-allocation-vcores + 2 + + + + yarn.scheduler.minimum-allocation-mb + 512 + + + + + yarn.scheduler.minimum-allocation-vcores + 1 + + + + yarn.timeline-service.address + __HDP_APP_TIMELINE_SERVER__:10200 + + + + yarn.timeline-service.bind-host + 0.0.0.0 + + + + yarn.timeline-service.client.max-retries + 30 + + + + yarn.timeline-service.client.retry-interval-ms + 1000 + + + + yarn.timeline-service.enabled + true + + + + yarn.timeline-service.generic-application-history.store-class + org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore + + + + yarn.timeline-service.http-authentication.simple.anonymous.allowed + true + + + + yarn.timeline-service.http-authentication.type + simple + + + + yarn.timeline-service.leveldb-state-store.path + /grid/0/hadoop/yarn/timeline + + + + yarn.timeline-service.leveldb-timeline-store.path + /grid/0/hadoop/yarn/timeline + + + + yarn.timeline-service.leveldb-timeline-store.read-cache-size + 104857600 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size + 10000 + + + + yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms + 300000 + + + + yarn.timeline-service.recovery.enabled + true + + + + yarn.timeline-service.state-store-class + org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore + + + + yarn.timeline-service.store-class + org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore + + + + yarn.timeline-service.ttl-enable + true + + + + yarn.timeline-service.ttl-ms + 2678400000 + + + + yarn.timeline-service.webapp.address + __HDP_APP_TIMELINE_SERVER__:8188 + + + + yarn.timeline-service.webapp.https.address + __HDP_APP_TIMELINE_SERVER__:8190 + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/core_hadoop/yarn.exclude @@ -0,0 +1 @@ + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml new file mode 100644 index 0000000..b021418 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/core-site.xml @@ -0,0 +1,147 @@ + + + + + fs.defaultFS + hdfs://__HDP_NAMENODE__:8020 + true + + + + fs.trash.interval + 360 + + + + ha.failover-controller.active-standby-elector.zk.op.retries + 120 + + + + hadoop.http.authentication.simple.anonymous.allowed + true + + + + hadoop.proxyuser.falcon.groups + users + + + + hadoop.proxyuser.falcon.hosts + * + + + + hadoop.proxyuser.hcat.groups + users + + + + hadoop.proxyuser.hcat.hosts + __HDP_WEBHCAT_SERVER__ + + + + hadoop.proxyuser.hdfs.groups + * + + + + hadoop.proxyuser.hdfs.hosts + * + + + + hadoop.proxyuser.hive.groups + users + + + + hadoop.proxyuser.hive.hosts + __HDP_HIVE_SERVER__ + + + + hadoop.proxyuser.oozie.groups + * + + + + hadoop.proxyuser.oozie.hosts + __HDP_OOZIE_SERVER__ + + + + hadoop.security.auth_to_local + DEFAULT + + + + hadoop.security.authentication + simple + + + + hadoop.security.authorization + false + + + + hadoop.security.key.provider.path + + + + + io.compression.codecs + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec + + + + io.file.buffer.size + 131072 + + + + io.serializations + org.apache.hadoop.io.serializer.WritableSerialization + + + + ipc.client.connect.max.retries + 50 + + + + ipc.client.connection.maxidletime + 30000 + + + + ipc.client.idlethreshold + 8000 + + + + ipc.server.tcpnodelay + true + + + + mapreduce.jobtracker.webinterface.trusted + false + + + + + + proxyuser_group + users + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..0911e46 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hadoop-metrics2-hbase.properties @@ -0,0 +1,50 @@ + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# See http://wiki.apache.org/hadoop/GangliaMetrics +# +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + + +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.slave.host.name=rohit-companionfiles-1.c.pramod-thangali.internal +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=10 +hbase.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=10 +jvm.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=10 +rpc.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period=10 +hbase.sink.timeline.collector=rohit-companionfiles-1.c.pramod-thangali.internal:6188 + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd new file mode 100644 index 0000000..9fb87d7 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.cmd @@ -0,0 +1,87 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Set environment variables here. + +@rem The java implementation to use. Java 1.7+ required. +@rem set JAVA_HOME=c:\apps\java + +@rem Extra Java CLASSPATH elements. Optional. +@rem set HBASE_CLASSPATH= + +@rem The maximum amount of heap to use. Default is left to JVM default. +@rem set HBASE_HEAPSIZE=1000 + +@rem Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +@rem offheap, set the value to "8G". +@rem set HBASE_OFFHEAPSIZE=1000 + +@rem For example, to allocate 8G of offheap, to 8G: +@rem etHBASE_OFFHEAPSIZE=8G + +@rem Extra Java runtime options. +@rem Below are what we set by default. May only work with SUN JVM. +@rem For more on why as well as other possible settings, +@rem see http://wiki.apache.org/hadoop/PerformanceTuning +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +@rem @rem See TestIPv6NIOServerSocketChannel. +set HBASE_OPTS="-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stack=true" + +@rem Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ +set HBASE_MASTER_OPTS=%HBASE_MASTER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" +set HBASE_REGIONSERVER_OPTS=%HBASE_REGIONSERVER_OPTS% "-XX:PermSize=128m" "-XX:MaxPermSize=128m" + +@rem Uncomment below to enable java garbage collection logging for the server-side processes +@rem this enables basic gc logging for the server processes to the .out file +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" "-XX:+UseGCLogFileRotation" "-XX:NumberOfGCLogFiles=1" "-XX:GCLogFileSize=512M" %HBASE_GC_OPTS% + +@rem Uncomment below to enable java garbage collection logging for the client processes in the .out file. +@rem set CLIENT_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +@rem set HBASE_USE_GC_LOGFILE=true + +@rem Uncomment and adjust to enable JMX exporting +@rem See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +@rem More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +@rem +@rem set HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false" "-Dcom.sun.management.jmxremote.authenticate=false" +@rem set HBASE_MASTER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10101" +@rem set HBASE_REGIONSERVER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10102" +@rem set HBASE_THRIFT_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10103" +@rem set HBASE_ZOOKEEPER_OPTS=%HBASE_JMX_BASE% -Dcom.sun.management.jmxremote.port=10104" + +@rem File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +@rem set HBASE_REGIONSERVERS=%HBASE_HOME%\conf\regionservers + +@rem Where log files are stored. $HBASE_HOME/logs by default. +@rem set HBASE_LOG_DIR=%HBASE_HOME%\logs + +@rem A string representing this instance of hbase. $USER by default. +@rem set HBASE_IDENT_STRING=%USERNAME% + +@rem Seconds to sleep between slave commands. Unset by default. This +@rem can be useful in large clusters, where, e.g., slave rsyncs can +@rem otherwise arrive faster than the master can service them. +@rem set HBASE_SLAVE_SLEEP=0.1 + +@rem Tell HBase whether it should manage it's own instance of Zookeeper or not. +@rem set HBASE_MANAGES_ZK=true diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh new file mode 100644 index 0000000..ce60e00 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-env.sh @@ -0,0 +1,65 @@ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +export JAVA_HOME=__JAVA_HOME__ + +# HBase Configuration directory +export HBASE_CONF_DIR=${HBASE_CONF_DIR:-__HBASE_CONF_DIR__} + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=${HBASE_CLASSPATH} + + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:__HBASE_LOG_DIR__/gc.log-`date +'%Y%m%d%H%M'`" +# Uncomment below to enable java garbage collection logging. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log" + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size +# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR=__HBASE_LOG_DIR__ + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR=/var/run/hbase + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + + + + +export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile=__HBASE_LOG_DIR__/hs_err_pid%p.log" +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx4096m $JDK_DEPENDED_OPTS" +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmn512m -XX:CMSInitiatingOccupancyFraction=70 -Xms1024m -Xmx8192m $JDK_DEPENDED_OPTS" diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml new file mode 100644 index 0000000..0392423 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-policy.xml @@ -0,0 +1,19 @@ + + + + + security.admin.protocol.acl + * + + + + security.client.protocol.acl + * + + + + security.masterregion.protocol.acl + * + + + \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml new file mode 100644 index 0000000..4c75e95 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hbase-site.xml @@ -0,0 +1,234 @@ + + + + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + + + hbase.bulkload.staging.dir + /apps/hbase/staging + + + + hbase.client.keyvalue.maxsize + 1048576 + + + + hbase.client.retries.number + 35 + + + + hbase.client.scanner.caching + 100 + + + + hbase.cluster.distributed + true + + + + hbase.coprocessor.master.classes + + + + + hbase.coprocessor.region.classes + org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint + + + + hbase.defaults.for.version.skip + true + + + + hbase.hregion.majorcompaction + 604800000 + + + + hbase.hregion.majorcompaction.jitter + 0.50 + + + + hbase.hregion.max.filesize + 10737418240 + + + + hbase.hregion.memstore.block.multiplier + 4 + + + + hbase.hregion.memstore.flush.size + 134217728 + + + + hbase.hregion.memstore.mslab.enabled + true + + + + hbase.hstore.blockingStoreFiles + 10 + + + + hbase.hstore.compaction.max + 10 + + + + hbase.hstore.compactionThreshold + 3 + + + + hbase.local.dir + ${hbase.tmp.dir}/local + + + + hbase.master.info.bindAddress + 0.0.0.0 + + + + hbase.master.info.port + 16010 + + + + hbase.master.port + 16000 + + + + hbase.region.server.rpc.scheduler.factory.class + + + + + hbase.regionserver.global.memstore.lowerLimit + 0.38 + + + + hbase.regionserver.global.memstore.size + 0.4 + + + + hbase.regionserver.global.memstore.upperLimit + 0.4 + + + + hbase.regionserver.handler.count + 30 + + + + hbase.regionserver.info.port + 16030 + + + + hbase.regionserver.port + 16020 + + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.WALCellCodec + + + + hbase.rootdir + hdfs://__HDP_NAMENODE__:8020/apps/hbase/data + + + + hbase.rpc.controllerfactory.class + + + + + hbase.rpc.protection + authentication + + + + hbase.rpc.timeout + 90000 + + + + hbase.security.authentication + simple + + + + hbase.security.authorization + false + + + + hbase.superuser + hbase + + + + hbase.tmp.dir + /tmp/hbase-${user.name} + + + + hbase.zookeeper.property.clientPort + 2181 + + + + hbase.zookeeper.quorum + __HDP_ZOOKEEPER_HOSTS__ + + + + hbase.zookeeper.useMulti + true + + + + hfile.block.cache.size + 0.40 + + + + phoenix.functions.allowUserDefinedFunctions + + + + + phoenix.query.timeoutMs + 60000 + + + + zookeeper.session.timeout + 90000 + + + + zookeeper.znode.parent + /hbase-unsecure + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml new file mode 100644 index 0000000..6367375 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/hdfs-site.xml @@ -0,0 +1,281 @@ + + + + + dfs.block.access.token.enable + true + + + + dfs.blockreport.initialDelay + 120 + + + + dfs.blocksize + 134217728 + + + + dfs.client.read.shortcircuit + true + + + + dfs.client.read.shortcircuit.streams.cache.size + 4096 + + + + dfs.client.retry.policy.enabled + false + + + + dfs.cluster.administrators + hdfs + + + + dfs.datanode.address + 0.0.0.0:50010 + + + + dfs.datanode.balance.bandwidthPerSec + 6250000 + + + + dfs.datanode.data.dir + /opt/data/hdfs/dn + + + + dfs.datanode.data.dir.perm + 750 + + + + dfs.datanode.du.reserved + 1073741824 + + + + dfs.datanode.failed.volumes.tolerated + 0 + + + + dfs.datanode.http.address + 0.0.0.0:50075 + + + + dfs.datanode.https.address + 0.0.0.0:50475 + + + + dfs.datanode.ipc.address + 0.0.0.0:8010 + + + + dfs.datanode.max.transfer.threads + 16384 + + + + dfs.domain.socket.path + /var/lib/hadoop-hdfs/dn_socket + + + + dfs.encryption.key.provider.uri + + + + + dfs.heartbeat.interval + 3 + + + + dfs.hosts.exclude + /etc/hadoop/conf/dfs.exclude + + + + dfs.http.policy + HTTP_ONLY + + + + dfs.https.port + 50470 + + + + dfs.journalnode.edits.dir + /hadoop/hdfs/journalnode + + + + dfs.journalnode.http-address + 0.0.0.0:8480 + + + + dfs.journalnode.https-address + 0.0.0.0:8481 + + + + dfs.namenode.accesstime.precision + 0 + + + + dfs.namenode.audit.log.async + true + + + + dfs.namenode.avoid.read.stale.datanode + true + + + + dfs.namenode.avoid.write.stale.datanode + true + + + + dfs.namenode.checkpoint.dir + __FS_CHECKPOINT_DIR__ + + + + dfs.namenode.checkpoint.edits.dir + ${dfs.namenode.checkpoint.dir} + + + + dfs.namenode.checkpoint.period + 21600 + + + + dfs.namenode.checkpoint.txns + 1000000 + + + + dfs.namenode.fslock.fair + false + + + + dfs.namenode.handler.count + 200 + + + + dfs.namenode.http-address + __HDP_NAMENODE__:50070 + true + + + + dfs.namenode.https-address + __HDP_NAMENODE__:50470 + + + + dfs.namenode.name.dir + __DFS_NAME_DIR__ + + + + dfs.namenode.name.dir.restore + true + + + + dfs.namenode.rpc-address + __HDP_NAMENODE__:8020 + + + + dfs.namenode.safemode.threshold-pct + 1 + + + + dfs.namenode.secondary.http-address + __HDP_SECONDARY_NAMENODE__:50090 + + + + dfs.namenode.stale.datanode.interval + 30000 + + + + dfs.namenode.startup.delay.block.deletion.sec + 3600 + + + + dfs.namenode.write.stale.datanode.ratio + 1.0f + + + + dfs.permissions.enabled + true + + + + dfs.permissions.superusergroup + hdfs + + + + dfs.replication + 3 + + + + dfs.replication.max + 50 + + + + dfs.support.append + true + true + + + + dfs.webhdfs.enabled + false + + + + fs.permissions.umask-mode + 022 + + + + nfs.exports.allowed.hosts + * rw + + + + nfs.file.dump.dir + /tmp/.hdfs-nfs + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties new file mode 100644 index 0000000..3737eac --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/log4j.properties @@ -0,0 +1,111 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=INFO +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers new file mode 100644 index 0000000..96dc196 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hbase/regionservers @@ -0,0 +1,3 @@ +__DCAE_CDAP_NODE0__ +__DCAE_CDAP_NODE1__ +__DCAE_CDAP_NODE2__ diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template new file mode 100644 index 0000000..fe47d94 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/beeline-log4j.properties.template @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, console + +######## console appender ######## +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties new file mode 100644 index 0000000..d65f9be --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/client.properties @@ -0,0 +1,5 @@ +# Generated by Apache Ambari. Thu Jun 18 00:59:57 2015 + +atlas.http.authentication.enabled=false +atlas.http.authentication.type=simple + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh new file mode 100644 index 0000000..fd99df0 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh @@ -0,0 +1,34 @@ + + if [ "$SERVICE" = "cli" ]; then + if [ -z "$DEBUG" ]; then + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit" + else + export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" + fi + fi + +# The heap size of the jvm stared by hive shell script can be controlled via: + +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +HADOOP_HOME=${HADOOP_HOME:-/usr/hdp/current/hadoop-client} + +# Hive Configuration Directory can be controlled by: +export HIVE_CONF_DIR=/usr/hdp/current/hive-client/conf + +# Folder containing extra libraries required for hive compilation/execution can be controlled by: +if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then + if [ -f "${HIVE_AUX_JARS_PATH}" ]; then + export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH} + elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar + fi +elif [ -d "/usr/hdp/current/hive-webhcat/share/hcatalog" ]; then + export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar +fi + +export METASTORE_PORT=9083 \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template new file mode 100644 index 0000000..91a2905 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-env.sh.template @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hive and Hadoop environment variables here. These variables can be used +# to control the execution of Hive. It should be used by admins to configure +# the Hive installation (so that users do not have to set environment variables +# or set command line parameters to get correct behavior). +# +# The hive service being invoked (CLI/HWI etc.) is available via the environment +# variable SERVICE + + +# Hive Client memory usage can be an issue if a large number of clients +# are running at the same time. The flags below have been useful in +# reducing memory usage: +# +# if [ "$SERVICE" = "cli" ]; then +# if [ -z "$DEBUG" ]; then +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit" +# else +# export HADOOP_OPTS="$HADOOP_OPTS -XX:NewRatio=12 -Xms10m -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit" +# fi +# fi + +# The heap size of the jvm stared by hive shell script can be controlled via: +# +# export HADOOP_HEAPSIZE=1024 +# +# Larger heap size may be required when running queries over large number of files or partitions. +# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be +# appropriate for hive server (hwi etc). + + +# Set HADOOP_HOME to point to a specific hadoop install directory +# HADOOP_HOME=${bin}/../../hadoop + +# Hive Configuration Directory can be controlled by: +# export HIVE_CONF_DIR= + +# Folder containing extra ibraries required for hive compilation/execution can be controlled by: +# export HIVE_AUX_JARS_PATH= diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties new file mode 100644 index 0000000..5b59a3f --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-exec-log4j.properties @@ -0,0 +1,79 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties + +hive.log.threshold=ALL +hive.root.logger=INFO,FA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.query.id=hadoop +hive.log.file=${hive.query.id}.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshhold=${hive.log.threshold} + +# +# File Appender +# + +log4j.appender.FA=org.apache.log4j.FileAppender +log4j.appender.FA.File=${hive.log.dir}/${hive.log.file} +log4j.appender.FA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,FA +log4j.category.Datastore=ERROR,FA +log4j.category.Datastore.Schema=ERROR,FA +log4j.category.JPOX.Datastore=ERROR,FA +log4j.category.JPOX.Plugin=ERROR,FA +log4j.category.JPOX.MetaData=ERROR,FA +log4j.category.JPOX.Query=ERROR,FA +log4j.category.JPOX.General=ERROR,FA +log4j.category.JPOX.Enhancer=ERROR,FA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties new file mode 100644 index 0000000..7861ba5 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-log4j.properties @@ -0,0 +1,89 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hive.log.threshold=ALL +hive.root.logger=INFO,DRFA +hive.log.dir=${java.io.tmpdir}/${user.name} +hive.log.file=hive.log + +# Define the root logger to the system property "hadoop.root.logger". +log4j.rootLogger=${hive.root.logger}, EventCounter + +# Logging Threshold +log4j.threshold=${hive.log.threshold} + +# +# Daily Rolling File Appender +# +# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files +# for different CLI session. +# +# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender + +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender + +log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +log4j.appender.console.encoding=UTF-8 + +#custom logging levels +#log4j.logger.xxx=DEBUG + +# +# Event Counter Appender +# Sends counts of logging messages at different severity levels to Hadoop Metrics. +# +log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter + + +log4j.category.DataNucleus=ERROR,DRFA +log4j.category.Datastore=ERROR,DRFA +log4j.category.Datastore.Schema=ERROR,DRFA +log4j.category.JPOX.Datastore=ERROR,DRFA +log4j.category.JPOX.Plugin=ERROR,DRFA +log4j.category.JPOX.MetaData=ERROR,DRFA +log4j.category.JPOX.Query=ERROR,DRFA +log4j.category.JPOX.General=ERROR,DRFA +log4j.category.JPOX.Enhancer=ERROR,DRFA + + +# Silence useless ZK logs +log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA +log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA \ No newline at end of file diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml new file mode 100644 index 0000000..a3c34e3 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/hive-site.xml @@ -0,0 +1,789 @@ + + + + + ambari.hive.db.schema.name + hive + + + + datanucleus.autoCreateSchema + false + + + + datanucleus.cache.level2.type + none + + + + hive.auto.convert.join + true + + + + hive.auto.convert.join.noconditionaltask + true + + + + hive.auto.convert.join.noconditionaltask.size + 894784504 + + + + hive.auto.convert.sortmerge.join + true + + + + hive.auto.convert.sortmerge.join.to.mapjoin + false + + + + hive.cbo.enable + true + + + + hive.cli.print.header + false + + + + hive.cluster.delegation.token.store.class + org.apache.hadoop.hive.thrift.ZooKeeperTokenStore + + + + hive.cluster.delegation.token.store.zookeeper.connectString + TODO_REPLACE_HOSTNAME:2181 + + + + hive.cluster.delegation.token.store.zookeeper.znode + /hive/cluster/delegation + + + + hive.cluster.name + companion + + + + hive.compactor.abortedtxn.threshold + 1000 + + + + hive.compactor.check.interval + 300L + + + + hive.compactor.delta.num.threshold + 10 + + + + hive.compactor.delta.pct.threshold + 0.1f + + + + hive.compactor.initiator.on + false + + + + hive.compactor.worker.threads + 0 + + + + hive.compactor.worker.timeout + 86400L + + + + hive.compute.query.using.stats + true + + + + hive.conf.restricted.list + hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role + + + + hive.convert.join.bucket.mapjoin.tez + false + + + + hive.default.fileformat + TextFile + + + + hive.enforce.bucketing + false + + + + hive.enforce.sorting + true + + + + hive.enforce.sortmergebucketmapjoin + true + + + + hive.exec.compress.intermediate + false + + + + hive.exec.compress.output + false + + + + hive.exec.dynamic.partition + true + + + + hive.exec.dynamic.partition.mode + strict + + + + hive.exec.failure.hooks + org.apache.hadoop.hive.ql.hooks.ATSHook + + + + hive.exec.max.created.files + 100000 + + + + hive.exec.max.dynamic.partitions + 5000 + + + + hive.exec.max.dynamic.partitions.pernode + 2000 + + + + hive.exec.orc.compression.strategy + SPEED + + + + hive.exec.orc.default.compress + ZLIB + + + + hive.exec.orc.default.stripe.size + 67108864 + + + + hive.exec.orc.encoding.strategy + SPEED + + + + hive.exec.parallel + false + + + + hive.exec.parallel.thread.number + 8 + + + + hive.exec.post.hooks + org.apache.hadoop.hive.ql.hooks.ATSHook, org.apache.atlas.hive.hook.HiveHook + + + + hive.exec.pre.hooks + org.apache.hadoop.hive.ql.hooks.ATSHook + + + + hive.exec.reducers.bytes.per.reducer + 67108864 + + + + hive.exec.reducers.max + 1009 + + + + hive.exec.scratchdir + /tmp/hive + + + + hive.exec.submit.local.task.via.child + true + + + + hive.exec.submitviachild + false + + + + hive.execution.engine + tez + + + + hive.fetch.task.aggr + false + + + + hive.fetch.task.conversion + more + + + + hive.fetch.task.conversion.threshold + 1073741824 + + + + hive.hook.dgi.url + http://TODO_REPLACE_HOSTNAME:21000 + + + + hive.limit.optimize.enable + true + + + + hive.limit.pushdown.memory.usage + 0.04 + + + + hive.map.aggr + true + + + + hive.map.aggr.hash.force.flush.memory.threshold + 0.9 + + + + hive.map.aggr.hash.min.reduction + 0.5 + + + + hive.map.aggr.hash.percentmemory + 0.5 + + + + hive.mapjoin.bucket.cache.size + 10000 + + + + hive.mapjoin.optimized.hashtable + true + + + + hive.mapred.reduce.tasks.speculative.execution + false + + + + hive.merge.mapfiles + true + + + + hive.merge.mapredfiles + false + + + + hive.merge.orcfile.stripe.level + true + + + + hive.merge.rcfile.block.level + true + + + + hive.merge.size.per.task + 256000000 + + + + hive.merge.smallfiles.avgsize + 16000000 + + + + hive.merge.tezfiles + false + + + + hive.metastore.authorization.storage.checks + false + + + + hive.metastore.cache.pinobjtypes + Table,Database,Type,FieldSchema,Order + + + + hive.metastore.client.connect.retry.delay + 5s + + + + hive.metastore.client.socket.timeout + 1800s + + + + hive.metastore.connect.retries + 24 + + + + hive.metastore.execute.setugi + true + + + + hive.metastore.failure.retries + 24 + + + + hive.metastore.kerberos.keytab.file + /etc/security/keytabs/hive.service.keytab + + + + hive.metastore.kerberos.principal + hive/_HOST@EXAMPLE.COM + + + + hive.metastore.pre.event.listeners + org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener + + + + hive.metastore.sasl.enabled + false + + + + hive.metastore.server.max.threads + 100000 + + + + hive.metastore.uris + thrift://TODO_REPLACE_HOSTNAME:9083 + + + + hive.metastore.warehouse.dir + /apps/hive/warehouse + + + + hive.optimize.bucketmapjoin + true + + + + hive.optimize.bucketmapjoin.sortedmerge + false + + + + hive.optimize.constant.propagation + true + + + + hive.optimize.index.filter + true + + + + hive.optimize.metadataonly + true + + + + hive.optimize.null.scan + true + + + + hive.optimize.reducededuplication + true + + + + hive.optimize.reducededuplication.min.reducer + 4 + + + + hive.optimize.sort.dynamic.partition + false + + + + hive.orc.compute.splits.num.threads + 10 + + + + hive.orc.splits.include.file.footer + false + + + + hive.prewarm.enabled + false + + + + hive.prewarm.numcontainers + 3 + + + + hive.security.authenticator.manager + org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator + + + + hive.security.authorization.enabled + false + + + + hive.security.authorization.manager + org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory + + + + hive.security.metastore.authenticator.manager + org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator + + + + hive.security.metastore.authorization.auth.reads + true + + + + hive.security.metastore.authorization.manager + org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider + + + + hive.server2.allow.user.substitution + true + + + + hive.server2.authentication + NONE + + + + hive.server2.authentication.spnego.keytab + HTTP/_HOST@EXAMPLE.COM + + + + hive.server2.authentication.spnego.principal + /etc/security/keytabs/spnego.service.keytab + + + + hive.server2.enable.doAs + true + + + + hive.server2.logging.operation.enabled + true + + + + hive.server2.logging.operation.log.location + ${system:java.io.tmpdir}/${system:user.name}/operation_logs + + + + hive.server2.support.dynamic.service.discovery + true + + + + hive.server2.table.type.mapping + CLASSIC + + + + hive.server2.tez.default.queues + default + + + + hive.server2.tez.initialize.default.sessions + false + + + + hive.server2.tez.sessions.per.default.queue + 1 + + + + hive.server2.thrift.http.path + cliservice + + + + hive.server2.thrift.http.port + 10001 + + + + hive.server2.thrift.max.worker.threads + 500 + + + + hive.server2.thrift.port + 10000 + + + + hive.server2.thrift.sasl.qop + auth + + + + hive.server2.transport.mode + binary + + + + hive.server2.use.SSL + false + + + + hive.server2.zookeeper.namespace + hiveserver2 + + + + hive.smbjoin.cache.rows + 10000 + + + + hive.stats.autogather + true + + + + hive.stats.dbclass + fs + + + + hive.stats.fetch.column.stats + true + + + + hive.stats.fetch.partition.stats + true + + + + hive.support.concurrency + false + + + + hive.tez.auto.reducer.parallelism + true + + + + hive.tez.container.size + 2560 + + + + hive.tez.cpu.vcores + -1 + + + + hive.tez.dynamic.partition.pruning + true + + + + hive.tez.dynamic.partition.pruning.max.data.size + 104857600 + + + + hive.tez.dynamic.partition.pruning.max.event.size + 1048576 + + + + hive.tez.input.format + org.apache.hadoop.hive.ql.io.HiveInputFormat + + + + hive.tez.java.opts + -server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps + + + + hive.tez.log.level + INFO + + + + hive.tez.max.partition.factor + 2.0 + + + + hive.tez.min.partition.factor + 0.25 + + + + hive.tez.smb.number.waves + 0.5 + + + + hive.txn.manager + org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager + + + + hive.txn.max.open.batch + 1000 + + + + hive.txn.timeout + 300 + + + + hive.user.install.directory + /user/ + + + + hive.vectorized.execution.enabled + true + + + + hive.vectorized.execution.reduce.enabled + false + + + + hive.vectorized.groupby.checkinterval + 4096 + + + + hive.vectorized.groupby.flush.percent + 0.1 + + + + hive.vectorized.groupby.maxentries + 100000 + + + + hive.zookeeper.client.port + 2181 + + + + hive.zookeeper.namespace + hive_zookeeper_namespace + + + + hive.zookeeper.quorum + TODO_REPLACE_HOSTNAME:2181 + + + + javax.jdo.option.ConnectionDriverName + com.mysql.jdbc.Driver + + + + javax.jdo.option.ConnectionURL + jdbc:mysql://TODO_REPLACE_HOSTNAME/hive?createDatabaseIfNotExist=true + + + + javax.jdo.option.ConnectionUserName + hive + + + + mapreduce.input.fileinputformat.split.minsize + 1000000 + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml new file mode 100644 index 0000000..bda842a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/ivysettings.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml new file mode 100644 index 0000000..6c71cae --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/hive/mapred-site.xml @@ -0,0 +1,224 @@ + + + + + mapreduce.admin.map.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} + + + + mapreduce.admin.reduce.child.java.opts + -server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version} + + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64 + + + + mapreduce.am.max-attempts + 2 + + + + mapreduce.application.classpath + $PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure + + + + mapreduce.application.framework.path + /hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework + + + + mapreduce.cluster.administrators + hadoop + + + + mapreduce.framework.name + yarn + + + + mapreduce.job.counters.max + 130 + + + + mapreduce.job.emit-timeline-data + false + + + + mapreduce.job.reduce.slowstart.completedmaps + 0.05 + + + + mapreduce.jobhistory.address + TODO_REPLACE_HOSTNAME:10020 + + + + mapreduce.jobhistory.bind-host + 0.0.0.0 + + + + mapreduce.jobhistory.done-dir + /mr-history/done + + + + mapreduce.jobhistory.intermediate-done-dir + /mr-history/tmp + + + + mapreduce.jobhistory.webapp.address + TODO_REPLACE_HOSTNAME:19888 + + + + mapreduce.map.java.opts + -Xmx2048m + + + + mapreduce.map.log.level + INFO + + + + mapreduce.map.memory.mb + 2560 + + + + mapreduce.map.output.compress + false + + + + mapreduce.map.sort.spill.percent + 0.7 + + + + mapreduce.map.speculative + false + + + + mapreduce.output.fileoutputformat.compress + false + + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + + mapreduce.reduce.input.buffer.percent + 0.0 + + + + mapreduce.reduce.java.opts + -Xmx4096m + + + + mapreduce.reduce.log.level + INFO + + + + mapreduce.reduce.memory.mb + 5120 + + + + mapreduce.reduce.shuffle.fetch.retry.enabled + 1 + + + + mapreduce.reduce.shuffle.fetch.retry.interval-ms + 1000 + + + + mapreduce.reduce.shuffle.fetch.retry.timeout-ms + 30000 + + + + mapreduce.reduce.shuffle.input.buffer.percent + 0.7 + + + + mapreduce.reduce.shuffle.merge.percent + 0.66 + + + + mapreduce.reduce.shuffle.parallelcopies + 30 + + + + mapreduce.reduce.speculative + false + + + + mapreduce.shuffle.port + 13562 + + + + mapreduce.task.io.sort.factor + 100 + + + + mapreduce.task.io.sort.mb + 1433 + + + + mapreduce.task.timeout + 300000 + + + + yarn.app.mapreduce.am.admin-command-opts + -Dhdp.version=${hdp.version} + + + + yarn.app.mapreduce.am.command-opts + -Xmx2048m -Dhdp.version=${hdp.version} + + + + yarn.app.mapreduce.am.log.level + INFO + + + + yarn.app.mapreduce.am.resource.mb + 2560 + + + + yarn.app.mapreduce.am.staging-dir + /user + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties new file mode 100644 index 0000000..83847de --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# Zookeeper connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zookeeper.connect=127.0.0.1:2181 + +# timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=6000 + +#consumer group id +group.id=test-consumer-group + +#consumer timeout +#consumer.timeout.ms=5000 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh new file mode 100644 index 0000000..cbfc937 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh @@ -0,0 +1,19 @@ + +#!/bin/bash + +# Set KAFKA specific environment variables here. + +# The java implementation to use. +export JAVA_HOME=/usr/jdk64/jdk1.8.0_40 +export PATH=$PATH:$JAVA_HOME/bin +export PID_DIR=/var/run/kafka +export LOG_DIR=/var/log/kafka +export KAFKA_KERBEROS_PARAMS= +# Add kafka sink to classpath and related depenencies +if [ -e "/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar" ]; then + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/* +fi +if [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then +. /etc/kafka/conf/kafka-ranger-env.sh +fi diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties new file mode 100644 index 0000000..2cba04d --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties @@ -0,0 +1,84 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# +kafka.logs.dir=logs + +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +#log4j.logger.kafka.perf=DEBUG, kafkaAppender +#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties new file mode 100644 index 0000000..47ae3e2 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.producer.ProducerConfig for more details + +############################# Producer Basics ############################# + +# list of brokers used for bootstrapping knowledge about the rest of the cluster +# format: host1:port1,host2:port2 ... +metadata.broker.list=localhost:9092 + +# name of the partitioner class for partitioning events; default partition spreads data randomly +#partitioner.class= + +# specifies whether the messages are sent asynchronously (async) or synchronously (sync) +producer.type=sync + +# specify the compression codec for all data generated: none, gzip, snappy, lz4. +# the old config values work as well: 0, 1, 2, 3 for none, gzip, snappy, lz4, respectively +compression.codec=none + +# message encoder +serializer.class=kafka.serializer.DefaultEncoder + +# allow topic level compression +#compressed.topics= + +############################# Async Producer ############################# +# maximum time, in milliseconds, for buffering data on the producer queue +#queue.buffering.max.ms= + +# the maximum size of the blocking queue for buffering on the producer +#queue.buffering.max.messages= + +# Timeout for event enqueue: +# 0: events will be enqueued immediately or dropped if the queue is full +# -ve: enqueue will block indefinitely if the queue is full +# +ve: enqueue will block up to this many milliseconds if the queue is full +#queue.enqueue.timeout.ms= + +# the number of messages batched at the producer +#batch.num.messages= diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties new file mode 100644 index 0000000..7dfb879 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties @@ -0,0 +1,70 @@ +auto.leader.rebalance.enable=true +kafka.ganglia.metrics.port=8671 +socket.send.buffer.bytes=102400 +message.max.bytes=1000000 +num.network.threads=3 +log.segment.bytes=1073741824 +kafka.ganglia.metrics.host=localhost +leader.imbalance.per.broker.percentage=10 +replica.lag.time.max.ms=10000 +authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer +num.io.threads=8 +offsets.retention.minutes=86400000 +fetch.purgatory.purge.interval.requests=10000 +offsets.topic.compression.codec=0 +replica.lag.max.messages=4000 +num.recovery.threads.per.data.dir=1 +log.dirs=/grid/0/kafka-logs +log.retention.bytes=-1 +producer.purgatory.purge.interval.requests=10000 +log.flush.scheduler.interval.ms=3000 +default.replication.factor=1 +replica.high.watermark.checkpoint.interval.ms=5000 +zookeeper.connect=TODO_REPLACE_HOSTNAME:2181 +controlled.shutdown.retry.backoff.ms=5000 +kafka.timeline.metrics.host=TODO_REPLACE_HOSTNAME +kafka.ganglia.metrics.group=kafka +kafka.timeline.metrics.reporter.sendInterval=5900 +num.partitions=1 +offsets.topic.segment.bytes=104857600 +controller.socket.timeout.ms=30000 +queued.max.requests=500 +broker.id=0 +offset.metadata.max.bytes=4096 +kafka.timeline.metrics.reporter.enabled=true +controlled.shutdown.max.retries=3 +min.insync.replicas=1 +offsets.commit.required.acks=-1 +replica.fetch.wait.max.ms=500 +controlled.shutdown.enable=true +log.roll.hours=168 +log.cleanup.interval.mins=10 +replica.socket.receive.buffer.bytes=65536 +kafka.ganglia.metrics.reporter.enabled=true +offsets.topic.num.partitions=50 +delete.topic.enable=false +offsets.load.buffer.size=5242880 +num.replica.fetchers=1 +socket.request.max.bytes=104857600 +kafka.timeline.metrics.maxRowCacheSize=10000 +kafka.timeline.metrics.port=6188 +compression.type=producer +zookeeper.sync.time.ms=2000 +zookeeper.connection.timeout.ms=6000 +socket.receive.buffer.bytes=102400 +controller.message.queue.size=10 +offsets.commit.timeout.ms=5000 +offsets.topic.replication.factor=3 +offsets.retention.check.interval.ms=600000 +log.flush.interval.ms=3000 +replica.fetch.max.bytes=1048576 +kafka.metrics.reporters=org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter +listeners=PLAINTEXT://TODO_REPLACE_HOSTNAME:6667 +replica.fetch.min.bytes=1 +replica.socket.timeout.ms=30000 +zookeeper.session.timeout.ms=30000 +auto.create.topics.enable=true +log.index.size.max.bytes=10485760 +leader.imbalance.check.interval.seconds=300 +log.index.interval.bytes=4096 +log.retention.hours=168 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties new file mode 100644 index 0000000..e0bbc13 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=logs/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=logs/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=logs/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=logs/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +log4j.logger.kafka.tools=DEBUG, kafkaAppender +log4j.logger.kafka.tools.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=TRACE, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties new file mode 100644 index 0000000..52f07c9 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties new file mode 100644 index 0000000..74cbf90 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# disable the per-ip limit on the number of connections since this is a non-production config +maxClientCnxns=0 diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl new file mode 100644 index 0000000..3e0216a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/configuration.xsl @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + +
namevaluedescription
+ + + + diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties new file mode 100644 index 0000000..20e1beb --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/log4j.properties @@ -0,0 +1,70 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# +# ZooKeeper Logging Configuration +# + +# DEFAULT: console appender only +log4j.rootLogger=INFO, CONSOLE + +# Example with rolling log file +#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=DEBUG +log4j.appender.ROLLINGFILE.File=zookeeper.log + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=zookeeper_trace.log + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg new file mode 100644 index 0000000..3d9911b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo.cfg @@ -0,0 +1,37 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +clientPort=2181 +initLimit=10 +autopurge.purgeInterval=24 +syncLimit=5 +tickTime=2000 +dataDir=__ZOOKEEPER_DATA_DIR__ +autopurge.snapRetainCount=30 +server.0=__DCAE_CDAP_NODE0__:2888:3888 +server.1=__DCAE_CDAP_NODE1__:2888:3888 +server.2=__DCAE_CDAP_NODE2__:2888:3888 + +# [DE239909] 3 Oct 2016 +maxClientCnxns=0 + diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg new file mode 100644 index 0000000..a5a2c0b --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zoo_sample.cfg @@ -0,0 +1,28 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +#autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +#autopurge.purgeInterval=1 diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd new file mode 100644 index 0000000..bad3b7a --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.cmd @@ -0,0 +1,28 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@rem Set environment variables here. + +@rem JVM parameters for both client and server +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +set JVMFLAGS=-Djava.net.preferIPv4Stack=true + +@rem Client specific JVM parameters +@rem set CLIENT_JVMFLAGS= + +@rem Server specific JVM parameters +@rem set SERVER_JVMFLAGS= diff --git a/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh new file mode 100644 index 0000000..8a545b3 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/zookeeper/zookeeper-env.sh @@ -0,0 +1,9 @@ + +export JAVA_HOME=__JAVA_HOME__ +export ZOOKEEPER_HOME=/usr/hdp/current/zookeeper-server +export ZOO_LOG_DIR=__ZOOKEEPER_LOG_DIR__ +export ZOOPIDFILE=__ZOOKEEPER_PID_DIR__/zookeeper_server.pid +export SERVER_JVMFLAGS=-Xmx1024m +export JAVA=$JAVA_HOME/bin/java +export CLASSPATH=$CLASSPATH:/usr/share/zookeeper/* + diff --git a/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl b/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl new file mode 100644 index 0000000..773f6ee --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-hosts-file.sh.tmpl @@ -0,0 +1,11 @@ +__HDP_NAMENODE__=__DCAE_CDAP_NODE0__ +__HDP_HBASE_MASTER__=__DCAE_CDAP_NODE1__ +__HDP_APP_TIMELINE_SERVER__=__DCAE_CDAP_NODE1__ +__HDP_HISTORYSERVER__=__DCAE_CDAP_NODE1__ +__HDP_RESOURCEMANAGER__=__DCAE_CDAP_NODE1__ +__HDP_SECONDARY_NAMENODE__=__DCAE_CDAP_NODE1__ +__HDP_ZOOKEEPER_HOSTS_WITH_PORT__=__DCAE_CDAP_NODE0__:2181,__DCAE_CDAP_NODE1__:2181,__DCAE_CDAP_NODE2__:2181 +__HDP_ZOOKEEPER_QUORUM__=__DCAE_CDAP_NODE0__,__DCAE_CDAP_NODE1__,__DCAE_CDAP_NODE2__:2181 +__HDP_ZOOKEEPER_HOSTS__=__DCAE_CDAP_NODE0__,__DCAE_CDAP_NODE1__,__DCAE_CDAP_NODE2__ +__CDAP_ROUTER_HOST__=__DCAE_CDAP_NODE2__ +__CDAP_KAFKA_HOST__=__DCAE_CDAP_NODE2__ diff --git a/cdap3vm/config/hostname-xref.txt.test b/cdap3vm/config/hostname-xref.txt.test new file mode 100644 index 0000000..faac169 --- /dev/null +++ b/cdap3vm/config/hostname-xref.txt.test @@ -0,0 +1,3 @@ +__DCAE_CDAP_NODE0__=c3p1.novalocal +__DCAE_CDAP_NODE1__=c3p0.novalocal +__DCAE_CDAP_NODE2__=c3p2.novalocal diff --git a/cdap3vm/config/services-setup-templates/cdap-setup.sh b/cdap3vm/config/services-setup-templates/cdap-setup.sh new file mode 100644 index 0000000..32c0891 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/cdap-setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Set up files and folders + +# Start CDAP services + diff --git a/cdap3vm/config/services-setup-templates/cdap-start.sh b/cdap3vm/config/services-setup-templates/cdap-start.sh new file mode 100644 index 0000000..5b7965b --- /dev/null +++ b/cdap3vm/config/services-setup-templates/cdap-start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export JAVA_HOME=__JAVA_HOME__ + +/etc/init.d/cdap-auth-server start +/etc/init.d/cdap-kafka-server start +/etc/init.d/cdap-master start +/etc/init.d/cdap-router start + +PATH=$PATH:__NODEJS_BIN__ +/etc/init.d/cdap-ui start diff --git a/cdap3vm/config/services-setup-templates/datanode-start.sh b/cdap3vm/config/services-setup-templates/datanode-start.sh new file mode 100644 index 0000000..c0d2076 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/datanode-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start datanode + diff --git a/cdap3vm/config/services-setup-templates/hbase-master-start.sh b/cdap3vm/config/services-setup-templates/hbase-master-start.sh new file mode 100644 index 0000000..62329cf --- /dev/null +++ b/cdap3vm/config/services-setup-templates/hbase-master-start.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh b/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh new file mode 100644 index 0000000..08eee7c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/hbase-regionserver-start.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +sleep 100 # Wait for all things to sync up? +__HDP_CURRENT_FOLDER__/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh b/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh new file mode 100644 index 0000000..d08237c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/job-history-setup-01-as-root.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +chown -R __SUPER_USER__:__HADOOP_USER__ __HDP_CURRENT_FOLDER__/hadoop-yarn*/bin/container-executor +chmod -R 6050 __HDP_CURRENT_FOLDER__/hadoop-yarn*/bin/container-executor diff --git a/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh b/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh new file mode 100644 index 0000000..5671c35 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/job-history-setup-02-as-hdfs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +hdfs dfs -mkdir -p /mr-history/tmp +hdfs dfs -chmod -R 1777 /mr-history/tmp + +hdfs dfs -mkdir -p /mr-history/done +hdfs dfs -chmod -R 1777 /mr-history/done +hdfs dfs -chown -R __MAPRED_USER__:__HDFS_USER__ /mr-history + +hdfs dfs -mkdir -p /app-logs +hdfs dfs -chmod -R 1777 /app-logs + +hdfs dfs -chown __YARN_USER__:__HDFS_USER__ /app-logs + + + diff --git a/cdap3vm/config/services-setup-templates/namenode-setup.sh b/cdap3vm/config/services-setup-templates/namenode-setup.sh new file mode 100644 index 0000000..5ba1297 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/namenode-setup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +__HDP_HADOOP_BIN__/hdfs namenode -format -force +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start namenode +__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ start datanode + +# Create HBASE related folders here +( +hdfs dfs -mkdir -p /apps/hbase/staging /apps/hbase/data + +hdfs dfs -chown hbase:hdfs /apps/hbase/staging /apps/hbase/data +hdfs dfs -chmod 711 /apps/hbase/staging +hdfs dfs -chmod 755 /apps/hbase/data + +hdfs dfs -chown hdfs:hdfs /apps/hbase +) & + +hdfs dfs -mkdir -p /hdp/apps/__HDP_VERSION__/mapreduce/ +hdfs dfs -put __HDP_CURRENT_FOLDER__/hadoop-client/mapreduce.tar.gz /hdp/apps/__HDP_VERSION__/mapreduce/ +hdfs dfs -chown -R __HDFS_USER__:__HADOOP_GROUP__ /hdp +hdfs dfs -chmod -R 555 /hdp/apps/__HDP_VERSION__/mapreduce +hdfs dfs -chmod 444 /hdp/apps/__HDP_VERSION__/mapreduce/mapreduce.tar.gz + +# We will try to set up general CDAP related stuff (cluster-prep) here +( +hdfs dfs -mkdir -p /user/yarn +hdfs dfs -chown yarn:yarn /user/yarn + +hdfs dfs -mkdir -p /cdap +hdfs dfs -chown yarn /cdap + +hdfs dfs -mkdir -p /cdap/tx.snapshot +hdfs dfs -chown yarn /cdap/tx.snapshot +) & diff --git a/cdap3vm/config/services-setup-templates/node-manager-start.sh b/cdap3vm/config/services-setup-templates/node-manager-start.sh new file mode 100644 index 0000000..6ebd914 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/node-manager-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ start nodemanager + diff --git a/cdap3vm/config/services-setup-templates/resource-manager-start.sh b/cdap3vm/config/services-setup-templates/resource-manager-start.sh new file mode 100644 index 0000000..85c660c --- /dev/null +++ b/cdap3vm/config/services-setup-templates/resource-manager-start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ start resourcemanager + diff --git a/cdap3vm/config/services-setup-templates/service-setup.sh b/cdap3vm/config/services-setup-templates/service-setup.sh new file mode 100644 index 0000000..a331c61 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/service-setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +MAINLOC=$(dirname $0) +setup_script="${1}" + +if [ -z "$2" ]; then + bash ${MAINLOC}/${setup_script} +else + su "$2" -c "bash ${MAINLOC}/${setup_script}" +fi diff --git a/cdap3vm/config/services-setup-templates/zookeeper-start.sh b/cdap3vm/config/services-setup-templates/zookeeper-start.sh new file mode 100644 index 0000000..2f974f1 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/zookeeper-start.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# ZooKeeper + +ACTION="start" + +export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__ +export ZOOCFG=zoo.cfg +source ${ZOOCFGDIR}/zookeeper-env.sh +__HDP_CURRENT_FOLDER__/zookeeper-server/bin/zkServer.sh "$ACTION" + + diff --git a/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh b/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh new file mode 100644 index 0000000..319fff3 --- /dev/null +++ b/cdap3vm/config/services-setup-templates/zookeeper-zkfc-start.sh @@ -0,0 +1,3 @@ +#!/bash + +__HDP_HADOOP_SBIN__/hadoop-daemon.sh start zkfc diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh new file mode 100644 index 0000000..c5c04ca --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N0.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + # [DE248720] start namenode on 00 + bash $MAINDIR/namenode.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + + wait_for_hbase_shell_OK + bash $MAINDIR/hbase-regionserver.sh $ACTION ;; + stop ) + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + # [DE256148] stop namenode on 00 + bash $MAINDIR/namenode.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh new file mode 100644 index 0000000..83145ad --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N1.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/resource-manager.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/job-history-server.sh $ACTION + + wait_for_namenode + bash $MAINDIR/hbase-master.sh $ACTION + bash $MAINDIR/hbase-regionserver.sh $ACTION ;; + stop ) + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/hbase-master.sh $ACTION + + bash $MAINDIR/job-history-server.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/resource-manager.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh b/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh new file mode 100644 index 0000000..7eebfa0 --- /dev/null +++ b/cdap3vm/config/services-templates/boot-time-cdap-vm-N2.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +\. $MAINDIR/utility-scripts.sh + +ACTION="$1" + +case "$ACTION" in + start ) + # [DE248724] Hadoop /var/run PID directories + bash $MAINDIR/create_pid_dirs.sh + bash $MAINDIR/zookeeper.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/node-manager.sh $ACTION + + wait_for_hbase_shell_OK + bash $MAINDIR/hbase-regionserver.sh $ACTION + bash $MAINDIR/cdap.sh $ACTION ;; + stop ) + bash $MAINDIR/cdap.sh $ACTION + bash $MAINDIR/hbase-regionserver.sh $ACTION + + bash $MAINDIR/node-manager.sh $ACTION + bash $MAINDIR/datanode.sh $ACTION + bash $MAINDIR/zookeeper.sh $ACTION ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac diff --git a/cdap3vm/config/services-templates/cdap-vm-services b/cdap3vm/config/services-templates/cdap-vm-services new file mode 100644 index 0000000..832847d --- /dev/null +++ b/cdap3vm/config/services-templates/cdap-vm-services @@ -0,0 +1,46 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# description: Starts and stops all CDAP related services +# +### BEGIN INIT INFO +# Provides: cdap-vm-services +# Short-Description: Hadoop and CDAP services for this CDAP node +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +MAINDIR=__SERVICE_CONFIG_FOLDER__ + +ACTION="$1" + +case "$ACTION" in + start|stop ) + bash $MAINDIR/boot-time-cdap-vm.sh "$ACTION" + ( cd /opt/app/dcae-controller-service-cdap-cluster-manager ; sudo su dcae -c "JAVA_HOME=/opt/app/java/jdk/jdk170 bin/manager.sh $ACTION" ) ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/cdap.sh b/cdap3vm/config/services-templates/cdap.sh new file mode 100644 index 0000000..24c79df --- /dev/null +++ b/cdap3vm/config/services-templates/cdap.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# CDAP Services + +ACTION="$1" + +case "$ACTION" in + start|stop|status ) + /etc/init.d/cdap-auth-server $ACTION + /etc/init.d/cdap-kafka-server $ACTION + /etc/init.d/cdap-master $ACTION + /etc/init.d/cdap-router $ACTION + /etc/init.d/cdap-ui $ACTION ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/datanode.sh b/cdap3vm/config/services-templates/datanode.sh new file mode 100644 index 0000000..29cbcc9 --- /dev/null +++ b/cdap3vm/config/services-templates/datanode.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Datanode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION datanode" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + + diff --git a/cdap3vm/config/services-templates/hbase-master.sh b/cdap3vm/config/services-templates/hbase-master.sh new file mode 100644 index 0000000..f928a0d --- /dev/null +++ b/cdap3vm/config/services-templates/hbase-master.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# HBase Master + +ACTION="$1" + +case "$ACTION" in + start|stop|restart ) + su - "__HBASE_USER__" -c "__HDP_CURRENT_FOLDER__/hbase-master/bin/hbase-daemon.sh $ACTION master" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + + diff --git a/cdap3vm/config/services-templates/hbase-regionserver.sh b/cdap3vm/config/services-templates/hbase-regionserver.sh new file mode 100644 index 0000000..83bba40 --- /dev/null +++ b/cdap3vm/config/services-templates/hbase-regionserver.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# HBase Region Server + +ACTION="$1" + +case "$ACTION" in + start|stop|restart ) + su - "__HBASE_USER__" -c "__HDP_CURRENT_FOLDER__/hbase-regionserver/bin/hbase-daemon.sh $ACTION regionserver" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/job-history-server.sh b/cdap3vm/config/services-templates/job-history-server.sh new file mode 100644 index 0000000..8f39aba --- /dev/null +++ b/cdap3vm/config/services-templates/job-history-server.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Job History Server + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su "__HDFS_USER__" -c "__HDP_CURRENT_FOLDER__/hadoop-mapreduce-historyserver/sbin/mr-jobhistory-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION historyserver" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + + + diff --git a/cdap3vm/config/services-templates/namenode.sh b/cdap3vm/config/services-templates/namenode.sh new file mode 100644 index 0000000..fac7bac --- /dev/null +++ b/cdap3vm/config/services-templates/namenode.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Namenode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION namenode" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/node-manager.sh b/cdap3vm/config/services-templates/node-manager.sh new file mode 100644 index 0000000..e2fd187 --- /dev/null +++ b/cdap3vm/config/services-templates/node-manager.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Yarn Node Manager + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __YARN_USER__ -c "__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION nodemanager" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/config/services-templates/resource-manager.sh b/cdap3vm/config/services-templates/resource-manager.sh new file mode 100644 index 0000000..e5b1a9f --- /dev/null +++ b/cdap3vm/config/services-templates/resource-manager.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Yarn Resource Manager + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __YARN_USER__ -c "__HDP_CURRENT_FOLDER__/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION resourcemanager" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + + + diff --git a/cdap3vm/config/services-templates/secondary-namenode.sh b/cdap3vm/config/services-templates/secondary-namenode.sh new file mode 100644 index 0000000..7fd5c3f --- /dev/null +++ b/cdap3vm/config/services-templates/secondary-namenode.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Secondary Namenode + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su __HDFS_USER__ -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh --config __HADOOP_CONF_DIR__ $ACTION secondarynamenode" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + diff --git a/cdap3vm/config/services-templates/service-start.sh b/cdap3vm/config/services-templates/service-start.sh new file mode 100644 index 0000000..a3094f9 --- /dev/null +++ b/cdap3vm/config/services-templates/service-start.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +service_name="${1%.sh}" + +username=${2:-$service_name} + +# Usage: $0 zookeeper will result in su zookeeper -c +su "$username" -c "bash ${MAINLOC}/${service_name}.sh start" diff --git a/cdap3vm/config/services-templates/utility-scripts.sh b/cdap3vm/config/services-templates/utility-scripts.sh new file mode 100644 index 0000000..5bfbe86 --- /dev/null +++ b/cdap3vm/config/services-templates/utility-scripts.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +wait_for_remote_service() { + remote_host="$1" + remote_port="$2" + sleep_time=${3:-30} # default of 30 seconds between retries + + # keep checking remote_host's remote_port every sleep_time seconds till we get a connection + while ( ! nc $remote_host $remote_port < /dev/null ); do sleep $sleep_time ; done +} + +wait_for_namenode() { + # keep checking namenode's port 8020 till it is up -- do it every 30 seconds + wait_for_remote_service __HDP_NAMENODE__ 8020 30 +} + +wait_for_hbase_shell_OK() { + # run hbase shell and see if we connect to hbase... Better than waiting for ports + while ( echo list | hbase shell 2>&1 | grep ^ERROR > /dev/null ); do + sleep 30 + done +} + diff --git a/cdap3vm/config/services-templates/zookeeper-zkfc.sh b/cdap3vm/config/services-templates/zookeeper-zkfc.sh new file mode 100644 index 0000000..d32973c --- /dev/null +++ b/cdap3vm/config/services-templates/zookeeper-zkfc.sh @@ -0,0 +1,33 @@ +#!/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper ZKFC + +ACTION="$1" + +case "$ACTION" in + start|stop ) + su "__HDFS_USER__" -c "__HDP_HADOOP_SBIN__/hadoop-daemon.sh $ACTION zkfc" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + diff --git a/cdap3vm/config/services-templates/zookeeper.sh b/cdap3vm/config/services-templates/zookeeper.sh new file mode 100644 index 0000000..666f40c --- /dev/null +++ b/cdap3vm/config/services-templates/zookeeper.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="$1" + +export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__ + +case "$ACTION" in + start|stop|restart|status ) + su "__ZOOKEEPER_USER__" -c "export ZOOCFGDIR=__ZOOKEEPER_CONF_DIR__; export ZOOCFG=zoo.cfg; source ${ZOOCFGDIR}/zookeeper-env.sh; __HDP_CURRENT_FOLDER__/zookeeper-server/bin/zkServer.sh $ACTION" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + + diff --git a/cdap3vm/install-steps/01-generate-host-ids-configs.sh b/cdap3vm/install-steps/01-generate-host-ids-configs.sh new file mode 100644 index 0000000..3a53078 --- /dev/null +++ b/cdap3vm/install-steps/01-generate-host-ids-configs.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +cd $(dirname $0) + +CFG=../config +HOST_XREF=${CFG}/hostname-xref.txt +CLUSTER_H_TEMPLATE=${CFG}/hadoop-cluster-hosts-file.sh.tmpl +CLUSTER_HOSTS=${CFG}/hadoop-cluster-hosts-file.sh + +HADOOP_CONF_FOLDER=../config +HADOOP_SCRIPT_FOLDER=../pkgs/services +HADOOP_SERVICE_SETUP_FOLDER=../pkgs/services-setup + +TEMPL_CDAP_CONF=../config/cdap-config-template +CDAP_CONF_FOLDER=../config/cdap-config + +# Hack to set system time to UTC first time +if [ -e /etc/localtime ]; then + rm /etc/localtime # remove it even if it is a symbolic link +fi +cp /usr/share/zoneinfo/UTC /etc/localtime +hwclock --systohc + +# Hack for 127.0.1.1 entry in /etc/hosts on Ubuntu images +# +sed '/^#/! {/127\.0\.1\.1/ s/^/# /}' -i /etc/hosts + +if [ "--no-hosts" = "$1" ]; then + echo "Using test hosts" + HOST_XREF=${HOST_XREF}.test +else + bash ./utils/generate-hosts.sh > ${HOST_XREF} +fi + +process_template() { + template="$1" + output_file="$2" + xref_file="$3" + python ./utils/replace-kv.py ${template} ${output_file} ${xref_file} +} + +process_hadoop_service_scripts() { + for x in "$@"; do + srcscript=${HADOOP_CONF_FOLDER}/services-templates/"${x}" + destscript=${HADOOP_SCRIPT_FOLDER}/"${x}" + xref_file=${MAIN_CLUSTER_CONFIG_XREF} + process_template ${srcscript} ${destscript} ${xref_file} + done +} + +process_hadoop_service_setup_scripts() { + for x in "$@"; do + srcscript=${HADOOP_CONF_FOLDER}/services-setup-templates/"${x}" + destscript=${HADOOP_SERVICE_SETUP_FOLDER}/"${x}" + xref_file=${MAIN_CLUSTER_CONFIG_XREF} + process_template ${srcscript} ${destscript} ${xref_file} + done +} + + +process_template ${CLUSTER_H_TEMPLATE} ${CLUSTER_HOSTS} ${HOST_XREF} + +# CLUSTER_HOSTS now has information on what NODE0, NODE1, etc., mean +# CLUSTER_HADOOP_CONF has information on folders, etc. +CLUSTER_HADOOP_CONF=${CFG}/hadoop-cluster-conf-file.sh + +MAIN_CLUSTER_CONFIG_XREF=${CFG}/main-conf.sh +# group them together +cat ${CLUSTER_HADOOP_CONF} ${CLUSTER_HOSTS} ${HOST_XREF} > ${MAIN_CLUSTER_CONFIG_XREF} + +# Create target configs from hadoop-config-templates +HCFG_TEMPL=${CFG}/hadoop-cluster-config-template + +HCFG=${CFG}/hadoop-cluster-config + +process_config_files() { + sfolder="$1" + mkdir -p ${HCFG}/${sfolder} + for x in ${HCFG_TEMPL}/${sfolder}/*; do + if [ -d $x ]; then continue; fi # skip folder + item=${x#$HCFG_TEMPL/} + template=${HCFG_TEMPL}/${item} + output_file=${HCFG}/${item} + process_template ${template} ${output_file} ${MAIN_CLUSTER_CONFIG_XREF} + done +} + +process_config_folder() { + sfolder="$1" + dfolder="$2" + mkdir -p ${dfolder} + for x in ${sfolder}/*; do + if [ -d $x ]; then continue; fi # skip folder + item=${x#$sfolder/} + template=${x} + output_file=${dfolder}/${item} + process_template ${template} ${output_file} ${MAIN_CLUSTER_CONFIG_XREF} + done +} + +process_config_files core_hadoop +process_config_files zookeeper +process_config_files hbase + +process_config_folder ${TEMPL_CDAP_CONF} ${CDAP_CONF_FOLDER} +process_config_folder ${TEMPL_CDAP_CONF}/common ${CDAP_CONF_FOLDER}/common + + +# TODO: We can simply process all files in the template folder for service_scripts? +process_hadoop_service_scripts \ + service-start.sh zookeeper.sh resource-manager.sh zookeeper-zkfc.sh \ + node-manager.sh namenode.sh datanode.sh secondary-namenode.sh \ + job-history-server.sh hbase-master.sh hbase-regionserver.sh \ + utility-scripts.sh cdap.sh cdap-vm-services \ + boot-time-cdap-vm-N0.sh boot-time-cdap-vm-N1.sh boot-time-cdap-vm-N2.sh + + +# TODO: We can simply process all files in the template folder for service_setup_scripts? +process_hadoop_service_setup_scripts \ + service-setup.sh namenode-setup.sh zookeeper-start.sh zookeeper-zkfc-start.sh \ + resource-manager-start.sh node-manager-start.sh datanode-start.sh \ + job-history-setup-01-as-root.sh job-history-setup-02-as-hdfs.sh \ + datanode-start.sh hbase-master-start.sh hbase-regionserver-start.sh \ + cdap-setup.sh cdap-start.sh + +chmod -R o+r ${HADOOP_SCRIPT_FOLDER}/.. + diff --git a/cdap3vm/install-steps/02-user-creation.sh b/cdap3vm/install-steps/02-user-creation.sh new file mode 100644 index 0000000..b5d6f03 --- /dev/null +++ b/cdap3vm/install-steps/02-user-creation.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Create user accounts for CDAP-Hadoop (3VM minimal) solution +# one argument after DESC creates a user x1:x1 +# additional "other-groups" argument creates a user x1:x1 -G y1,y2,... +# additional groups have to be specified in a comma-separated format without spaces + +while read desc user othergrouplist; do + if [ -z "${othergrouplist}" ]; then + useradd $user -m + else + useradd $user -m + usermod -a -G ${othergrouplist} $user + fi +done << __EOF__ +HADOOP hadoop +HDFS hdfs hadoop +HBase hbase hadoop +YARN yarn hadoop +MapReduce mapred hadoop +ZooKeeper zookeeper hadoop +CDAP cdap +DCAE dcae hdfs +__EOF__ + diff --git a/cdap3vm/install-steps/04-folder-creation.sh b/cdap3vm/install-steps/04-folder-creation.sh new file mode 100644 index 0000000..59ca5cf --- /dev/null +++ b/cdap3vm/install-steps/04-folder-creation.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +\. ../config/main-conf.sh +\. ./utils/folder-creation-utils.sh + +HADOOP_SERVICE_SETUP_FOLDER=../pkgs/services-setup + +NODETYPE=N$(get_cdap_host_id) +NODETYPE=${1:-$NODETYPE} + +X=$NODETYPE +if [[ "$X" != "N0" && "$X" != "N1" && "$X" != "N2" && "$X" != "NTEST" ]]; then + exit -1 +fi + +if [ "$NODETYPE" == "NTEST" ]; then TEST_ZOOKEEPER_ID=0; fi + +NODESTATE="$2" # default is inst, but we may have a spare type + +CFG=../config +HCFG_TEMPL=${CFG}/hadoop-cluster-config-template + +TEMPL_CDAP_CONF=../config/cdap-config-template +CDAP_CONF_FOLDER=../config/cdap-config + +HADOOP_CONF_FOLDER=../config/hadoop-cluster-config +HADOOP_SCRIPT_FOLDER=../pkgs/services + +IS_INITIAL=1 # Fresh cluster +MAXHOSTS=3 + +if [[ -n "$NODESTATE" && "inst" != "$NODESTATE" ]]; then + IS_ISINITIAL= +fi + +process_generic_node + +# [DE248724] configure tmpfiles.d +cp ../pkgs/ubuntu-files/cdap-hadoop-run.conf /usr/lib/tmpfiles.d/ +chmod 644 /usr/lib/tmpfiles.d/cdap-hadoop-run.conf + +cp ../pkgs/ubuntu-files/create_pid_dirs.sh "${__SERVICE_CONFIG_FOLDER__}"/ +chmod 755 "${__SERVICE_CONFIG_FOLDER__}"/create_pid_dirs.sh + +cp $HADOOP_SCRIPT_FOLDER/boot-time-cdap-vm-${NODETYPE}.sh "${__SERVICE_CONFIG_FOLDER__}"/boot-time-cdap-vm.sh +cp $HADOOP_SCRIPT_FOLDER/cdap-vm-services /etc/init.d +chmod +x /etc/init.d/cdap-vm-services + +# Ubuntu service enable: +# +update-rc.d cdap-vm-services defaults + +\. ./utils/cdap-nodetype-${NODETYPE}.sh diff --git a/cdap3vm/install-steps/install-cdap-pkgs.sh b/cdap3vm/install-steps/install-cdap-pkgs.sh new file mode 100755 index 0000000..c4c7ffa --- /dev/null +++ b/cdap3vm/install-steps/install-cdap-pkgs.sh @@ -0,0 +1,49 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +export JAVA_HOME=/opt/app/java/jdk/jdk170 +PATH=$JAVA_HOME/bin:$PATH + +## Assumption: Hadoop is already installed and configured +# +# hdfs.namespace is /cdap and the default property hdfs.user is yarn +# +sudo su hdfs -c "JAVA_HOME=/opt/app/java/jdk/jdk170 hadoop fs -mkdir -p /cdap && hadoop fs -chown yarn /cdap" + +# create a tx.snapshot subdirectory +# +sudo su hdfs -c "JAVA_HOME=/opt/app/java/jdk/jdk170 hadoop fs -mkdir -p /cdap/tx.snapshot && hadoop fs -chown yarn /cdap/tx.snapshot" + +## Configure apt-get repo for CDAP 3.5.x: +# +echo "Configuring CDAP 3.5 apt-get repo..." +sudo wget -O /etc/apt/sources.list.d/cask.list http://repository.cask.co/ubuntu/precise/amd64/cdap/3.5/cask.list + +## Add the Cask Public GPG Key: +# +echo "Adding the Cask public GPG key to apt..." +wget -O - http://repository.cask.co/ubuntu/precise/amd64/cdap/3.5/pubkey.gpg | sudo apt-key add - + +## Update apt cache: +# +echo "apt-get update..." +sudo apt-get -y update + +## Install CDAP packages: +echo "installing cdap packages..." +sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get -y install cdap-gateway cdap-kafka cdap-master cdap-security cdap-ui cdap-cli diff --git a/cdap3vm/install-steps/install-hortonworks-hadoop.sh b/cdap3vm/install-steps/install-hortonworks-hadoop.sh new file mode 100755 index 0000000..9e73eeb --- /dev/null +++ b/cdap3vm/install-steps/install-hortonworks-hadoop.sh @@ -0,0 +1,61 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +export JAVA_HOME=/opt/app/java/jdk/jdk170 + +## Configure apt-get repo for HortonWorks: +# +echo "Configuring HortonWorks apt-get repo..." +sudo wget http://public-repo-1.hortonworks.com/HDP/ubuntu14/2.x/updates/2.4.3.0/hdp.list -O /etc/apt/sources.list.d/hdp.list + +## Add the BigTop Public GPG Key: +# +# echo "Adding the HortonWorks public GPG key to apt..." +sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 07513CAD + +## Update apt cache: +# +echo "apt-get update..." +sudo apt-get -y update + +## List Hadoop packages: +# +sudo apt-cache search hadoop +sudo apt-cache search hbase +sudo apt-cache search zookeeper + +## Install HortonWorks Hadoop packages: +# +# sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get install hadoop\* + +# sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get --print-uris install \ + +sudo JAVA_HOME=/opt/app/java/jdk/jdk170 apt-get -y install \ + hadoop hadoop-client hadoop-hdfs hadoop-hdfs-datanode hadoop-hdfs-namenode \ + hadoop-hdfs-zkfc hadoop-mapreduce hadoop-mapreduce-historyserver \ + hadoop-yarn hadoop-yarn-nodemanager \ + hbase hbase-master hbase-regionserver \ + zookeeper libhdfs0 + +## Fix file permissions for domain sockets +# +sudo chown hdfs:hadoop /var/lib/hadoop-hdfs +sudo chown mapred:hadoop /var/lib/hadoop-mapreduce +sudo chown yarn:hadoop /var/lib/hadoop-yarn +sudo chmod 775 /var/lib/hadoop-mapreduce + diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh new file mode 100644 index 0000000..d55f6ae --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N0.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/node-manager.sh + +wait_for_hbase_shell_OK +bash $MAINDIR/hbase-regionserver.sh + diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh new file mode 100644 index 0000000..d8affb9 --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N1.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/resource-manager.sh +bash $MAINDIR/node-manager.sh +bash $MAINDIR/job-history-server.sh + +wait_for_namenode +bash $MAINDIR/hbase-master.sh +bash $MAINDIR/hbase-regionserver.sh diff --git a/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh new file mode 100644 index 0000000..eb9c9f3 --- /dev/null +++ b/cdap3vm/install-steps/utils/boot-time-cdap-vm-N2.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +MAINDIR=/etc/hadoop/service_scripts/ + +\. $MAINDIR/utility-scripts.sh + +bash $MAINDIR/zookeeper.sh start +bash $MAINDIR/datanode.sh +bash $MAINDIR/node-manager.sh + +wait_for_hbase_shell_OK +bash $MAINDIR/hbase-regionserver.sh +bash $MAINDIR/cdap.sh diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh new file mode 100644 index 0000000..f6140a4 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N0.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_namenode # will do datanode +inst_yarn_nodemanager + +wait_for_hbase_shell_OK +inst_hbase_regionserver + +# inst_cdap # inst_cdap will start CDAP services diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh new file mode 100644 index 0000000..4e87124 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N1.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_datanode +inst_yarn_resourcemanager +inst_yarn_nodemanager +inst_job_history_server + +wait_for_namenode +sleep 20 # for namenode to create required folders; also hbase master needs namenode + +inst_hbase_master +inst_hbase_regionserver + +# inst_cdap # inst_cdap will start CDAP services diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh b/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh new file mode 100644 index 0000000..52ef5fc --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-N2.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_datanode +inst_yarn_nodemanager + +wait_for_hbase_shell_OK +inst_hbase_regionserver + +inst_cdap diff --git a/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh b/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh new file mode 100644 index 0000000..0ae4b3d --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-nodetype-NTEST.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Customization for this node -- after generic node is processed + +inst_namenode # will do datanode +inst_yarn_node_manager + +inst_cdap +inst_yarn_resource_manager +inst_job_history_server +inst_hbase_master +inst_hbase_regionserver diff --git a/cdap3vm/install-steps/utils/cdap-vm-services b/cdap3vm/install-steps/utils/cdap-vm-services new file mode 100644 index 0000000..281cd36 --- /dev/null +++ b/cdap3vm/install-steps/utils/cdap-vm-services @@ -0,0 +1,42 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# description: Starts and stops the master service +# +### BEGIN INIT INFO +# Provides: cdap-vm-services +# Short-Description: Hadoop and CDAP services for this CDAP node +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Required-Start: $syslog $remote_fs +# Required-Stop: $syslog $remote_fs +# Should-Start: +# Should-Stop: +### END INIT INFO + +MAINDIR=/etc/hadoop/service-scripts/ + +if [ "start" = "$1" ]; then + bash $MAINDIR/boot-time-cdap-vm.sh + ( cd /opt/app/dcae-controller-service-cdap-cluster-manager ; sudo su dcae -c "JAVA_HOME=/opt/app/java/jdk/jdk170 bin/manager.sh start" ) +fi + + + diff --git a/cdap3vm/install-steps/utils/folder-creation-utils.sh b/cdap3vm/install-steps/utils/folder-creation-utils.sh new file mode 100644 index 0000000..8a8f82a --- /dev/null +++ b/cdap3vm/install-steps/utils/folder-creation-utils.sh @@ -0,0 +1,241 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +copy_hadoop_conf_files() { + srcfolder=${HADOOP_CONF_FOLDER}/"$1" + destfolder="$2" + cp -a $srcfolder/* $destfolder/ +} + +copy_cdap_conf_files() { + srcfolder=${CDAP_CONF_FOLDER} + destfolder="${__CDAP_CONF_DIR__}" + cp -a $srcfolder/cdap-* $srcfolder/log* $destfolder/ + # Hack for a common.sh file -- TODO: Make it more general + cp -a $srcfolder/common/common.sh ${__CDAP_INST_FOLDER__}/ui/bin/common.sh +} + + +setup_hadoop_service_scripts() { + for x in "$@"; do + srcscript=${HADOOP_SCRIPT_FOLDER}/"${x}" + destscript="${__SERVICE_CONFIG_FOLDER__}/${x}" + cp -a $srcscript $destscript + done +} + +run_service_setup_script() { + script="$1" + user="$2" + runner=${HADOOP_SERVICE_SETUP_FOLDER}/service-setup.sh + srcscript=${HADOOP_SERVICE_SETUP_FOLDER}/"${script}" + bash ${runner} ${script} ${user} # should not give full path +} + +create_folder() { + # Create a folder with a group and mode + name=$1 # if there are spaces, there will be multiple folders + user="$2" + group="$3" + mode="$4" + + mkdir -p $name # if there are spaces, there will be multiple folders + chown -R "$user":"$group" $name + chmod -R "$mode" $name +} + +inst_namenode() { + if [ -n "$IS_INITIAL" ]; then + rm -rf ${__DFS_NAME_DIR__} # TODO: Move away from force delete? + create_folder ${__DFS_NAME_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + run_service_setup_script namenode-setup.sh ${__HDFS_USER__} + setup_hadoop_service_scripts namenode.sh + else + exit -1; # We do not know what to do + fi +} + +inst_secondary_namenode() { + if [ -n "$IS_INITIAL" ]; then + create_folder ${__FS_CHECKPOINT_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + setup_hadoop_service_scripts secondary-namenode.sh + else + exit -1; # We do not know what to do + fi +} + +inst_yarn_basic() { + create_folder ${__YARN_LOCAL_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + create_folder ${__YARN_LOCAL_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 +} + +inst_datanode_basic() { + create_folder ${__HDFS_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + rm -rf ${__DFS_DATA_DIR__} # TODO: Move away from force delete? + create_folder ${__DFS_DATA_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 750 + setup_hadoop_service_scripts datanode.sh +} + +inst_datanode() { + run_service_setup_script datanode-start.sh ${__HDFS_USER__} +} + +inst_hbase_basic() { + create_folder ${__HBASE_LOG_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750 + create_folder ${__HBASE_PID_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750 + rm -rf ${__HBASE_CONF_DIR__} + create_folder ${__HBASE_CONF_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 755 + copy_hadoop_conf_files hbase ${__HBASE_CONF_DIR__} +} + +inst_hbase_master() { + run_service_setup_script hbase-master-start.sh ${__HBASE_USER__} + setup_hadoop_service_scripts hbase-master.sh +} + +inst_hbase_regionserver() { + run_service_setup_script hbase-regionserver-start.sh ${__HBASE_USER__} + setup_hadoop_service_scripts hbase-regionserver.sh +} + +inst_core_files() { + # For all nodes + # YARN logs + create_folder ${__YARN_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # HDFS PID folder + create_folder ${__HDFS_PID_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # YARN Node Manager Recovery Directory + create_folder ${__YARN_NODEMANAGER_RECOVERY_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # YARN PID folder + create_folder ${__YARN_PID_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755 + # JobHistory server logs + create_folder ${__MAPRED_LOG_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755 + # JobHistory PID folder + create_folder ${__MAPRED_PID_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755 + # hadoop conf dir + rm -rf ${__HADOOP_CONF_DIR__} + create_folder ${__HADOOP_CONF_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # dfs.exclude -- before things are created + touch ${__HADOOP_CONF_DIR__}/dfs.exclude + copy_hadoop_conf_files core_hadoop ${__HADOOP_CONF_DIR__} +} + +inst_yarn_nodemanager() { + run_service_setup_script node-manager-start.sh ${__YARN_USER__} + setup_hadoop_service_scripts node-manager.sh +} + +inst_yarn_resourcemanager() { + run_service_setup_script resource-manager-start.sh ${__YARN_USER__} + setup_hadoop_service_scripts resource-manager.sh +} + +inst_job_history_server() { + run_service_setup_script job-history-setup-01-as-root.sh # no need for username then + run_service_setup_script job-history-setup-02-as-hdfs.sh ${__HDFS_USER__} + setup_hadoop_service_scripts job-history-server.sh +} + +inst_cdap() { + # note: cluster-prep is done along with namenode + run_service_setup_script cdap-setup.sh # as root + setup_hadoop_service_scripts cdap.sh + copy_cdap_conf_files + run_service_setup_script cdap-start.sh # as root +} + +inst_zookeeper() { + # Zookeeper data folder + create_folder ${__ZOOKEEPER_DATA_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + # Zookeeper logs + create_folder ${__ZOOKEEPER_LOG_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + create_folder ${__ZKFC_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755 + # Zookeeper PID folder + create_folder ${__ZOOKEEPER_PID_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + # Put the Zookeeper ID into its folder + get_zookeeper_id > ${__ZOOKEEPER_DATA_DIR__}/myid + # Clean up Zookeeper conf folder + rm -rf ${__ZOOKEEPER_CONF_DIR__} + create_folder ${__ZOOKEEPER_CONF_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755 + chmod 755 ${__ZOOKEEPER_CONF_DIR__}/.. # Parent folder of zookeeper + # Copy zookeeper files + copy_hadoop_conf_files zookeeper ${__ZOOKEEPER_CONF_DIR__} + run_service_setup_script zookeeper-start.sh ${__ZOOKEEPER_USER__} + run_service_setup_script zookeeper-zkfc-start.sh ${__HDFS_USER__} +} + +get_cdap_host_id() { + # we will set this based on the hostname + base=$(hostname -s) + base=$(echo $base | sed 's/^.*[^0-9]\([0-9][0-9]*\)/\1/g') + base=${base: -1} # get last digit. TODO: Need to make it scale? + echo $base +} + +get_zookeeper_id() { + echo $(get_cdap_host_id) +} + +get_zookeeper_id_single_digit() { + base=$(get_cdap_host_id) + echo ${base: -1} +} + +process_generic_node() { + + create_folder ${__SERVICE_CONFIG_FOLDER__} root root 755 + + inst_core_files # config files, creating, copying etc. + setup_hadoop_service_scripts utility-scripts.sh + setup_hadoop_service_scripts service-start.sh + setup_hadoop_service_scripts zookeeper.sh + setup_hadoop_service_scripts zookeeper-zkfc.sh + + inst_zookeeper + inst_datanode_basic + inst_hbase_basic + inst_yarn_basic +} + +wait_for_remote_service() { + remote_host="$1" + remote_port="$2" + sleep_time=${3:-30} # default of 30 seconds between retries + + # keep checking remote_host's remote_port every sleep_time seconds till we get a connection + while ( ! nc $remote_host $remote_port < /dev/null ); do sleep $sleep_time ; done +} + +wait_for_namenode() { + # keep checking namenode's port 8020 till it is up -- do it every 30 seconds + wait_for_remote_service ${__HDP_NAMENODE__} 8020 30 +} + +wait_for_hbase_master_start() { + # keep checking hbase master's port 16000 till it is up -- do it every 30 seconds + wait_for_remote_service ${__HDP_HBASE_MASTER__} 16000 30 +} + +wait_for_hbase_shell_OK() { + # run hbase shell and see if we connect to hbase... Better than waiting for ports + while ( echo list | hbase shell 2>&1 | grep ^ERROR > /dev/null ); do + sleep 30 + done +} diff --git a/cdap3vm/install-steps/utils/generate-hosts.sh b/cdap3vm/install-steps/utils/generate-hosts.sh new file mode 100644 index 0000000..b080a1e --- /dev/null +++ b/cdap3vm/install-steps/utils/generate-hosts.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAXHOSTID=2 + +fullhostname=$(hostname -f) +basehostname=${fullhostname%%.*} +restname=${fullhostname#*.} + +basenum=$(echo $base | sed 's/^.*[^0-9]\([0-9][0-9]*\)/\1/g') +basenum=${basenum##0*} # remove leading zeros + +if [[ "${basenum}" -ge 0 && "${basenum}" -le ${MAXHOSTS} ]]; then + for x in $(seq 0 $MAXHOSTID); do + echo "__DCAE_CDAP_NODE${x}__=${basehostname::-1}${x}" + # echo "__DCAE_CDAP_NODE${x}__=${basehostname::-1}${x}.${restname}" + # echo "_CLUSTER_SERVER_PREFIX__0${x}=${basehostname::-1}${x}.${restname}" + done +fi + + diff --git a/cdap3vm/install-steps/utils/replace-kv.py b/cdap3vm/install-steps/utils/replace-kv.py new file mode 100644 index 0000000..acd9782 --- /dev/null +++ b/cdap3vm/install-steps/utils/replace-kv.py @@ -0,0 +1,63 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +from __future__ import print_function +import os +import re +import sys + +""" +Replace patterns in a file with the corresponding replacements. + +Args: OriginalFile OutFile XRefFile +""" + + +def process_kv_file(fname): + """ + Read a simple properties file, X=Y type; file may contain comment lines and blanks. Return a dict + Leading comments are '^\s*#', while blank lines aare '^\s*$' + Key cannot contain '=', but value can + """ + ignore_pattern = re.compile(r'^\s*(#|$)') + with open(fname) as fid: + all_lines = [ line.strip() for line in fid if not re.match(ignore_pattern, line) ] + return dict( line.partition('=')[::2] for line in all_lines ) + + +def replace_kv(fname, outfname, *xref_files): + """ + Read a file, and perform multiple search replace using key-values in xref. + Keys have to be simple keys + """ + xref = {} + for xref_file in xref_files: + xref.update(process_kv_file(xref_file)) + pattern = re.compile("|".join(xref.keys())) + with open(outfname, 'w') as outf: + with open(fname) as fid: + all_text = fid.read() + outf.write(pattern.sub(lambda m: xref[m.group(0)], all_text)) + + +if __name__ == "__main__": + if len(sys.argv) >= 3: + replace_kv(*sys.argv[1:]) + else: + print("Usage: {} <{}> <{}> <{}> [{}]".format( + sys.argv[0], "InputFile", "OutPutFile", "XREF-DictFile[s]", "..."), file=sys.stderr); diff --git a/cdap3vm/install.sh b/cdap3vm/install.sh new file mode 100644 index 0000000..d2947ac --- /dev/null +++ b/cdap3vm/install.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# Default will be a production installation +# Any test installation requires passing arguments to different steps + +cd $(dirname $0)/install-steps + +bash 01-generate-host-ids-configs.sh +bash 02-user-creation.sh +#bash 03-hadoop-rpms.sh +bash install-hortonworks-hadoop.sh +bash install-cdap-pkgs.sh +bash 04-folder-creation.sh + diff --git a/cdap3vm/pkgs/services-setup/cdap-setup.sh b/cdap3vm/pkgs/services-setup/cdap-setup.sh new file mode 100644 index 0000000..137e9e3 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/cdap-setup.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# CDAP setup on all machines! + +echo "Should have already installed CDAP RPMs" + +exit + +RPM_DIR="../installation-steps" + +( + cd $RPM_DIR/04-cdap-pkgs + rpm -Uvh * +) + + diff --git a/cdap3vm/pkgs/services-setup/datanode-start.sh b/cdap3vm/pkgs/services-setup/datanode-start.sh new file mode 100644 index 0000000..77a0f4e --- /dev/null +++ b/cdap3vm/pkgs/services-setup/datanode-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + diff --git a/cdap3vm/pkgs/services-setup/hbase-master-start.sh b/cdap3vm/pkgs/services-setup/hbase-master-start.sh new file mode 100644 index 0000000..3e0db6b --- /dev/null +++ b/cdap3vm/pkgs/services-setup/hbase-master-start.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh b/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh new file mode 100644 index 0000000..259091f --- /dev/null +++ b/cdap3vm/pkgs/services-setup/hbase-regionserver-start.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +sleep 100 # Wait for all things to sync up? +/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh b/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh new file mode 100644 index 0000000..9c5e992 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/job-history-setup-01-as-root.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +chown -R root:hadoop /usr/hdp/current/hadoop-yarn*/bin/container-executor +chmod -R 6050 /usr/hdp/current/hadoop-yarn*/bin/container-executor diff --git a/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh b/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh new file mode 100644 index 0000000..4b365f1 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/job-history-setup-02-as-hdfs.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +hdfs dfs -mkdir -p /mr-history/tmp +hdfs dfs -chmod -R 1777 /mr-history/tmp + +hdfs dfs -mkdir -p /mr-history/done +hdfs dfs -chmod -R 1777 /mr-history/done +hdfs dfs -chown -R mapred:hdfs /mr-history + +hdfs dfs -mkdir -p /app-logs +hdfs dfs -chmod -R 1777 /app-logs + +hdfs dfs -chown yarn:hdfs /app-logs + + + diff --git a/cdap3vm/pkgs/services-setup/namenode-setup.sh b/cdap3vm/pkgs/services-setup/namenode-setup.sh new file mode 100644 index 0000000..54c9d1c --- /dev/null +++ b/cdap3vm/pkgs/services-setup/namenode-setup.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/bin/hdfs namenode -format -force +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + +sleep 30 # Wait for services to come up? + +hdfs dfs -mkdir -p /hdp/apps/2.3.2.0-2950/mapreduce/ +hdfs dfs -put /usr/hdp/current/hadoop-client/mapreduce.tar.gz /hdp/apps/2.3.2.0-2950/mapreduce/ +hdfs dfs -chown -R hdfs:hadoop /hdp +hdfs dfs -chmod -R 555 /hdp/apps/2.3.2.0-2950/mapreduce +hdfs dfs -chmod 444 /hdp/apps/2.3.2.0-2950/mapreduce/mapreduce.tar.gz + +# We will try to set up general CDAP related stuff (cluster-prep) here +( +hdfs dfs -mkdir -p /user/yarn +hdfs dfs -chown yarn /user/yarn +hdfs dfs -chgrp yarn /user/yarn + +hdfs dfs -mkdir -p /cdap +hdfs dfs -chown yarn /cdap + +hdfs dfs -mkdir -p /cdap/tx.snapshot +hdfs dfs -chown yarn /cdap/tx.snapshot +) & diff --git a/cdap3vm/pkgs/services-setup/node-manager-start.sh b/cdap3vm/pkgs/services-setup/node-manager-start.sh new file mode 100644 index 0000000..d0403ee --- /dev/null +++ b/cdap3vm/pkgs/services-setup/node-manager-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start nodemanager + diff --git a/cdap3vm/pkgs/services-setup/resource-manager-start.sh b/cdap3vm/pkgs/services-setup/resource-manager-start.sh new file mode 100644 index 0000000..bf8366f --- /dev/null +++ b/cdap3vm/pkgs/services-setup/resource-manager-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start resourcemanager + diff --git a/cdap3vm/pkgs/services-setup/service-setup.sh b/cdap3vm/pkgs/services-setup/service-setup.sh new file mode 100644 index 0000000..e3733f0 --- /dev/null +++ b/cdap3vm/pkgs/services-setup/service-setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +setup_script="${1}" + +if [ -z "$2" ]; then + bash ${MAINLOC}/${setup_script} +else + su "$2" -c "bash ${MAINLOC}/${setup_script}" +fi diff --git a/cdap3vm/pkgs/services-setup/zookeeper-start.sh b/cdap3vm/pkgs/services-setup/zookeeper-start.sh new file mode 100644 index 0000000..64c4b7d --- /dev/null +++ b/cdap3vm/pkgs/services-setup/zookeeper-start.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="start" + +export ZOOCFGDIR=/etc/zookeeper/conf +export ZOOCFG=zoo.cfg +source ${ZOOCFGDIR}/zookeeper-env.sh +/usr/hdp/current/zookeeper-server/bin/zkServer.sh "$ACTION" + + diff --git a/cdap3vm/pkgs/services/datanode.sh b/cdap3vm/pkgs/services/datanode.sh new file mode 100644 index 0000000..77a0f4e --- /dev/null +++ b/cdap3vm/pkgs/services/datanode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start datanode + diff --git a/cdap3vm/pkgs/services/hbase-master.sh b/cdap3vm/pkgs/services/hbase-master.sh new file mode 100644 index 0000000..3e0db6b --- /dev/null +++ b/cdap3vm/pkgs/services/hbase-master.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hbase-master/bin/hbase-daemon.sh start master diff --git a/cdap3vm/pkgs/services/hbase-regionserver.sh b/cdap3vm/pkgs/services/hbase-regionserver.sh new file mode 100644 index 0000000..259091f --- /dev/null +++ b/cdap3vm/pkgs/services/hbase-regionserver.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +sleep 100 # Wait for all things to sync up? +/usr/hdp/current/hbase-regionserver/bin/hbase-daemon.sh start regionserver + diff --git a/cdap3vm/pkgs/services/job-history-server.sh b/cdap3vm/pkgs/services/job-history-server.sh new file mode 100644 index 0000000..de1f30a --- /dev/null +++ b/cdap3vm/pkgs/services/job-history-server.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-mapreduce-historyserver/sbin/mr-jobhistory-daemon.sh --config /etc/hadoop/conf start historyserver + diff --git a/cdap3vm/pkgs/services/namenode.sh b/cdap3vm/pkgs/services/namenode.sh new file mode 100644 index 0000000..2814f28 --- /dev/null +++ b/cdap3vm/pkgs/services/namenode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode + diff --git a/cdap3vm/pkgs/services/node-manager.sh b/cdap3vm/pkgs/services/node-manager.sh new file mode 100644 index 0000000..d0403ee --- /dev/null +++ b/cdap3vm/pkgs/services/node-manager.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-nodemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start nodemanager + diff --git a/cdap3vm/pkgs/services/resource-manager.sh b/cdap3vm/pkgs/services/resource-manager.sh new file mode 100644 index 0000000..bf8366f --- /dev/null +++ b/cdap3vm/pkgs/services/resource-manager.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-yarn-resourcemanager/sbin/yarn-daemon.sh --config /etc/hadoop/conf start resourcemanager + diff --git a/cdap3vm/pkgs/services/secondary-namenode.sh b/cdap3vm/pkgs/services/secondary-namenode.sh new file mode 100644 index 0000000..2c6dc0f --- /dev/null +++ b/cdap3vm/pkgs/services/secondary-namenode.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +/usr/hdp/current/hadoop-client/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start secondarynamenode + diff --git a/cdap3vm/pkgs/services/service-start.sh b/cdap3vm/pkgs/services/service-start.sh new file mode 100644 index 0000000..a3094f9 --- /dev/null +++ b/cdap3vm/pkgs/services/service-start.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +MAINLOC=$(dirname $0) +service_name="${1%.sh}" + +username=${2:-$service_name} + +# Usage: $0 zookeeper will result in su zookeeper -c +su "$username" -c "bash ${MAINLOC}/${service_name}.sh start" diff --git a/cdap3vm/pkgs/services/zookeeper.sh b/cdap3vm/pkgs/services/zookeeper.sh new file mode 100644 index 0000000..9f86d9d --- /dev/null +++ b/cdap3vm/pkgs/services/zookeeper.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# ZooKeeper + +ACTION="$1" + +case "$ACTION" in + start|stop|restart|status ) + export ZOOCFGDIR=/etc/zookeeper/conf + export ZOOCFG=zoo.cfg + source ${ZOOCFGDIR}/zookeeper-env.sh + /usr/hdp/current/zookeeper-server/bin/zkServer.sh "$ACTION" ;; + * ) + echo "Usage: $0 " + exit -1 ;; +esac + diff --git a/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf b/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf new file mode 100644 index 0000000..770689c --- /dev/null +++ b/cdap3vm/pkgs/ubuntu-files/cdap-hadoop-run.conf @@ -0,0 +1,34 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# this file is part of dcae-cdap-small-hadoop +# it belongs in /usr/lib/tmpfiles.d +# +# [DE248724] Hadoop /var/run PID directories not persisted between +# reboots on CentOS7 +# +# +d /var/run/cdap 0755 cdap cdap - +d /var/run/hbase 0750 hbase hadoop - +d /var/run/zookeeper 0755 zookeeper hadoop - +d /var/run/hadoop 0755 root root - +d /var/run/hadoop/hdfs 0755 hdfs hadoop - +d /var/run/hadoop/yarn 0755 yarn hadoop - +d /var/run/hadoop/yarn/yarn 0755 yarn yarn - +d /var/run/hadoop/mapred 0755 mapred hadoop - +d /var/run/hadoop/mapreduce 0775 mapred hadoop - diff --git a/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh b/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh new file mode 100644 index 0000000..2812b31 --- /dev/null +++ b/cdap3vm/pkgs/ubuntu-files/create_pid_dirs.sh @@ -0,0 +1,48 @@ +# ============LICENSE_START========================================== +# =================================================================== +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# =================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property. + +# [DE248724] Hadoop /var/run PID directories not persisted between +# reboots +# Ubuntu 14 does not actually use /usr/lib/tmpfs.d, so need to +# handle creating PID dirs in init scripts. + +# zookeeper: +sudo mkdir -p -m0755 /var/run/zookeeper +sudo chown zookeeper:hadoop /var/run/zookeeper + +# hbase: +sudo mkdir -p -m0750 /var/run/hbase +sudo chown hbase:hadoop /var/run/hbase + +# hdfs: +sudo mkdir -p -m0755 /var/run/hadoop/hdfs +sudo chown hdfs:hadoop /var/run/hadoop/hdfs + +# yarn: +sudo mkdir -p -m0755 /var/run/hadoop/yarn +sudo chown yarn:hadoop /var/run/hadoop/yarn +sudo mkdir -p -m0755 /var/run/hadoop/yarn/yarn +sudo chown yarn:hadoop /var/run/hadoop/yarn/yarn + +# mapred: +sudo mkdir -p -m0755 /var/run/hadoop/mapred +sudo chown mapred:hadoop /var/run/hadoop/mapred + +# mapreduce: +sudo mkdir -p -m0755 /var/run/hadoop/mapreduce +sudo chown mapred:hadoop /var/run/hadoop/mapreduce -- cgit 1.2.3-korg