Init dcae.apod.cdap

Change-Id: I0d1f1ba04f8ba7a2a276e4e382eb07ae9d6a7652
Signed-off-by: lj1412 <lji@research.att.com>

1 files changed, 241 insertions, 0 deletions

diff --git a/cdap3vm/install-steps/utils/folder-creation-utils.sh b/cdap3vm/install-steps/utils/folder-creation-utils.sh
new file mode 100644
index 0000000..8a8f82a
--- /dev/null
+++ b/cdap3vm/install-steps/utils/folder-creation-utils.sh
@@ -0,0 +1,241 @@
+#!/bin/bash
+
+# ============LICENSE_START==========================================
+# ===================================================================
+# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END============================================
+# ECOMP and OpenECOMP are trademarks and service marks of AT&T Intellectual Property.
+
+copy_hadoop_conf_files() {
+   srcfolder=${HADOOP_CONF_FOLDER}/"$1"
+   destfolder="$2"
+   cp -a $srcfolder/* $destfolder/
+}
+
+copy_cdap_conf_files() {
+   srcfolder=${CDAP_CONF_FOLDER}
+   destfolder="${__CDAP_CONF_DIR__}"
+   cp -a $srcfolder/cdap-* $srcfolder/log* $destfolder/
+   # Hack for a common.sh file -- TODO: Make it more general
+   cp -a $srcfolder/common/common.sh ${__CDAP_INST_FOLDER__}/ui/bin/common.sh
+}
+
+   
+setup_hadoop_service_scripts() {
+   for x in "$@"; do
+       srcscript=${HADOOP_SCRIPT_FOLDER}/"${x}"
+       destscript="${__SERVICE_CONFIG_FOLDER__}/${x}"
+       cp -a $srcscript $destscript
+   done
+}
+  
+run_service_setup_script() {
+   script="$1"
+   user="$2"
+   runner=${HADOOP_SERVICE_SETUP_FOLDER}/service-setup.sh
+   srcscript=${HADOOP_SERVICE_SETUP_FOLDER}/"${script}"
+   bash ${runner} ${script} ${user} # should not give full path
+}
+ 
+create_folder() {
+    # Create a folder with a group and mode
+    name=$1  # if there are spaces, there will be multiple folders
+    user="$2"
+    group="$3"
+    mode="$4"
+
+   mkdir -p $name  # if there are spaces, there will be multiple folders
+   chown -R "$user":"$group" $name
+   chmod -R "$mode" $name
+}
+
+inst_namenode() {
+   if [ -n "$IS_INITIAL" ]; then
+      rm -rf ${__DFS_NAME_DIR__}  # TODO: Move away from force delete?
+      create_folder ${__DFS_NAME_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+      run_service_setup_script namenode-setup.sh ${__HDFS_USER__} 
+      setup_hadoop_service_scripts namenode.sh
+   else
+      exit -1; # We do not know what to do
+   fi
+}
+
+inst_secondary_namenode() {
+   if [ -n "$IS_INITIAL" ]; then
+      create_folder ${__FS_CHECKPOINT_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+      setup_hadoop_service_scripts secondary-namenode.sh
+   else
+      exit -1; # We do not know what to do
+   fi
+}
+
+inst_yarn_basic() {
+    create_folder ${__YARN_LOCAL_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755
+    create_folder ${__YARN_LOCAL_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755
+}   
+
+inst_datanode_basic() {
+    create_folder ${__HDFS_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+    rm -rf ${__DFS_DATA_DIR__}  # TODO: Move away from force delete?
+    create_folder ${__DFS_DATA_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 750
+    setup_hadoop_service_scripts datanode.sh
+}   
+
+inst_datanode() {
+    run_service_setup_script datanode-start.sh ${__HDFS_USER__}
+}
+
+inst_hbase_basic() {
+    create_folder ${__HBASE_LOG_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750
+    create_folder ${__HBASE_PID_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 750
+    rm -rf ${__HBASE_CONF_DIR__}
+    create_folder ${__HBASE_CONF_DIR__} ${__HBASE_USER__} ${__HADOOP_GROUP__} 755
+    copy_hadoop_conf_files hbase ${__HBASE_CONF_DIR__}
+}
+
+inst_hbase_master() {
+    run_service_setup_script hbase-master-start.sh ${__HBASE_USER__} 
+    setup_hadoop_service_scripts hbase-master.sh 
+}
+
+inst_hbase_regionserver() {
+    run_service_setup_script hbase-regionserver-start.sh ${__HBASE_USER__} 
+    setup_hadoop_service_scripts hbase-regionserver.sh 
+}
+
+inst_core_files() {
+   # For all  nodes
+   # YARN logs
+   create_folder ${__YARN_LOG_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755
+   # HDFS PID folder
+   create_folder ${__HDFS_PID_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+   # YARN Node Manager Recovery Directory
+   create_folder ${__YARN_NODEMANAGER_RECOVERY_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755
+   # YARN PID folder
+   create_folder ${__YARN_PID_DIR__} ${__YARN_USER__} ${__HADOOP_GROUP__} 755
+   # JobHistory server logs
+   create_folder ${__MAPRED_LOG_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755
+   # JobHistory PID folder
+   create_folder ${__MAPRED_PID_DIR__} ${__MAPRED_USER__} ${__HADOOP_GROUP__} 755
+   # hadoop conf dir
+   rm -rf ${__HADOOP_CONF_DIR__}
+   create_folder ${__HADOOP_CONF_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+   # dfs.exclude -- before things are created
+   touch ${__HADOOP_CONF_DIR__}/dfs.exclude 
+   copy_hadoop_conf_files core_hadoop ${__HADOOP_CONF_DIR__}
+}
+
+inst_yarn_nodemanager() {
+   run_service_setup_script node-manager-start.sh ${__YARN_USER__}
+   setup_hadoop_service_scripts node-manager.sh
+}
+
+inst_yarn_resourcemanager() {
+   run_service_setup_script resource-manager-start.sh ${__YARN_USER__}
+   setup_hadoop_service_scripts resource-manager.sh 
+}
+
+inst_job_history_server() {
+   run_service_setup_script job-history-setup-01-as-root.sh # no need for username then
+   run_service_setup_script job-history-setup-02-as-hdfs.sh ${__HDFS_USER__}
+   setup_hadoop_service_scripts job-history-server.sh  
+}
+
+inst_cdap() {
+   # note: cluster-prep is done along with namenode
+   run_service_setup_script cdap-setup.sh  # as root
+   setup_hadoop_service_scripts cdap.sh
+   copy_cdap_conf_files 
+   run_service_setup_script cdap-start.sh  # as root
+}
+
+inst_zookeeper() {
+   # Zookeeper data folder
+   create_folder ${__ZOOKEEPER_DATA_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755
+   # Zookeeper logs
+   create_folder ${__ZOOKEEPER_LOG_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755
+   create_folder ${__ZKFC_LOG_DIR__} ${__HDFS_USER__} ${__HADOOP_GROUP__} 755
+   # Zookeeper PID folder
+   create_folder ${__ZOOKEEPER_PID_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755
+   # Put the Zookeeper ID into its folder
+   get_zookeeper_id > ${__ZOOKEEPER_DATA_DIR__}/myid
+   # Clean up Zookeeper conf folder
+   rm -rf ${__ZOOKEEPER_CONF_DIR__}
+   create_folder ${__ZOOKEEPER_CONF_DIR__} ${__ZOOKEEPER_USER__} ${__HADOOP_GROUP__} 755
+   chmod 755 ${__ZOOKEEPER_CONF_DIR__}/..  # Parent folder of zookeeper
+   # Copy zookeeper files
+   copy_hadoop_conf_files zookeeper ${__ZOOKEEPER_CONF_DIR__}
+   run_service_setup_script zookeeper-start.sh ${__ZOOKEEPER_USER__} 
+   run_service_setup_script zookeeper-zkfc-start.sh ${__HDFS_USER__} 
+}
+
+get_cdap_host_id() {
+   # we will set this based on the hostname
+   base=$(hostname -s)
+   base=$(echo $base | sed 's/^.*[^0-9]\([0-9][0-9]*\)/\1/g')
+   base=${base: -1}  # get last digit. TODO: Need to make it scale?
+   echo $base
+}
+
+get_zookeeper_id() {
+   echo $(get_cdap_host_id) 
+}
+
+get_zookeeper_id_single_digit() {
+   base=$(get_cdap_host_id)
+   echo ${base: -1}
+}
+
+process_generic_node() {
+
+  create_folder ${__SERVICE_CONFIG_FOLDER__} root root 755
+
+  inst_core_files  # config files, creating, copying etc.
+  setup_hadoop_service_scripts utility-scripts.sh
+  setup_hadoop_service_scripts service-start.sh
+  setup_hadoop_service_scripts zookeeper.sh 
+  setup_hadoop_service_scripts zookeeper-zkfc.sh 
+
+  inst_zookeeper
+  inst_datanode_basic
+  inst_hbase_basic
+  inst_yarn_basic
+} 
+
+wait_for_remote_service() {
+  remote_host="$1"
+  remote_port="$2"
+  sleep_time=${3:-30} # default of 30 seconds between retries
+
+  # keep checking remote_host's remote_port every sleep_time seconds till we get a connection
+  while ( ! nc $remote_host $remote_port < /dev/null ); do sleep $sleep_time ; done 
+}
+
+wait_for_namenode() {
+  # keep checking namenode's port 8020 till it is up -- do it every 30 seconds
+  wait_for_remote_service ${__HDP_NAMENODE__} 8020 30 
+}
+
+wait_for_hbase_master_start() {
+  # keep checking hbase master's port 16000 till it is up -- do it every 30 seconds
+  wait_for_remote_service ${__HDP_HBASE_MASTER__} 16000 30 
+}
+
+wait_for_hbase_shell_OK() {
+  # run hbase shell and see if we connect to hbase... Better than waiting for ports
+  while ( echo list | hbase shell 2>&1 | grep ^ERROR > /dev/null ); do
+    sleep 30
+  done
+}