diff options
Diffstat (limited to 'src/main/scripts')
| -rw-r--r-- | src/main/scripts/common_functions.sh | 1 | ||||
| -rw-r--r-- | src/main/scripts/dupeTool.sh | 5 | ||||
| -rw-r--r-- | src/main/scripts/dynamicPayloadArchive.sh | 2 | ||||
| -rw-r--r-- | src/main/scripts/extract-events.sh | 20 | ||||
| -rw-r--r-- | src/main/scripts/getDslResult.sh | 151 | ||||
| -rw-r--r-- | src/main/scripts/resend-dmaap-events.sh | 147 | ||||
| -rw-r--r-- | src/main/scripts/run_SendDeleteMigrationNotification.sh | 2 | ||||
| -rw-r--r-- | src/main/scripts/run_SendMigrationNotification.sh | 4 |
8 files changed, 257 insertions, 75 deletions
diff --git a/src/main/scripts/common_functions.sh b/src/main/scripts/common_functions.sh index f279334..1fac7ce 100644 --- a/src/main/scripts/common_functions.sh +++ b/src/main/scripts/common_functions.sh @@ -16,7 +16,6 @@ check_user(){ # Sources the profile and sets the project home source_profile(){ - . /etc/profile.d/aai.sh PROJECT_HOME=/opt/app/aai-graphadmin } diff --git a/src/main/scripts/dupeTool.sh b/src/main/scripts/dupeTool.sh index 350b0bd..d4cdb9c 100644 --- a/src/main/scripts/dupeTool.sh +++ b/src/main/scripts/dupeTool.sh @@ -68,6 +68,9 @@ COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) start_date; check_user; source_profile; + +export JAVA_PRE_OPTS=${JAVA_PRE_OPTS:--Xms6g -Xmx6g}; + execute_spring_jar org.onap.aai.dbgen.DupeTool ${PROJECT_HOME}/resources/dupeTool-logback.xml "$@" end_date; -exit 0
\ No newline at end of file +exit 0 diff --git a/src/main/scripts/dynamicPayloadArchive.sh b/src/main/scripts/dynamicPayloadArchive.sh index 87cce13..89cf990 100644 --- a/src/main/scripts/dynamicPayloadArchive.sh +++ b/src/main/scripts/dynamicPayloadArchive.sh @@ -29,7 +29,7 @@ COMMON_ENV_PATH=$( cd "$(dirname "$0")" ; pwd -P ) . ${COMMON_ENV_PATH}/common_functions.sh -. /etc/profile.d/aai.sh + PROJECT_HOME=/opt/app/aai-graphadmin PROGNAME=$(basename $0) diff --git a/src/main/scripts/extract-events.sh b/src/main/scripts/extract-events.sh new file mode 100644 index 0000000..5a2bb3b --- /dev/null +++ b/src/main/scripts/extract-events.sh @@ -0,0 +1,20 @@ +#!/bin/bash +usage(){ + echo "Usage $0 input-file output-file event-type"; +} + +if [ "${#}" -lt 3 ]; then + usage; + exit -1 +fi; + +input_file=$1 +output_file=$2 +event_type=$3 + +grep "|${event_type}|" ${input_file} > ${output_file}.1 +sed -i -e '/InvokeReturn/s/^.*$//g' ${output_file}.1 +sed -i '/^$/d' ${output_file}.1 +cat ${output_file}.1 | awk -F '|' '{print $29}' > ${output_file} +rm ${output_file}.1 +exit 0 diff --git a/src/main/scripts/getDslResult.sh b/src/main/scripts/getDslResult.sh new file mode 100644 index 0000000..78286cd --- /dev/null +++ b/src/main/scripts/getDslResult.sh @@ -0,0 +1,151 @@ +#!/bin/ksh + +### +# ============LICENSE_START======================================================= +# org.onap.aai +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +### + +# + +display_usage() { + cat <<EOF + Usage: $0 [options] + + 1. Usage: getDslResult.sh <base-path or optional host url> <optional input-json-filepath> <optional -XFROMAPPID> <optional -XTRANSID> + 2. This script requires one argument, a base-path + 3. Example for basepath: aai/{version}\ + 4. Adding the optional input-json-payload replaces the default dsl payload with the contents of the input-file + 5. The query result is returned in the file resources/migration-input-files/dslResults.json +EOF +} +if [ $# -eq 0 ]; then + display_usage + exit 1 +fi + +RESOURCE="dsl?format=resource_and_url&nodesOnly=true" + +BASEPATH=$1 +if [ -z $BASEPATH ]; then + echo "base-path parameter is missing" + echo "usage: $0 <base-path>" + exit 1 +fi + + +PROJECT_HOME=/opt/app/aai-graphadmin +RESULTDIR=$PROJECT_HOME/logs/data/migration-input-files + +if [ ! -d ${RESULTDIR} ]; then + mkdir -p ${RESULTDIR} + chown aaiadmin:aaiadmin ${RESULTDIR} + chmod u+w ${RESULTDIR} +fi + +RESULTPATH=$RESULTDIR/dslResults.json + +JSONFILE=$2 +TEMPFILE=/tmp/dslResult-temp.json +if [ -z $JSONFILE ]; then + JSONFILE=$TEMPFILE + echo "{ \"dsl\":\"l3-network('network-type','Tenant Network')>vlan-tag>vlan-range*\" }" > $JSONFILE +fi + +echo `date` " Starting $0 for resource $RESOURCE" + + +XFROMAPPID="AAI-TOOLS" +XTRANSID=`uuidgen` + +if [ ! -z "$3" ]; then + XFROMAPPID=$3 +fi + +if [ ! -z "$4" ]; then + XTRANSID=$4 +fi + +userid=$( id | cut -f2 -d"(" | cut -f1 -d")" ) +if [ "${userid}" != "aaiadmin" ]; then + echo "You must be aaiadmin to run $0. The id used $userid." + exit 1 +fi + +prop_file=$PROJECT_HOME/resources/etc/appprops/aaiconfig.properties +log_dir=$PROJECT_HOME/logs/misc +today=$(date +\%Y-\%m-\%d) + +RETURNRESPONSE=true + +MISSING_PROP=false +RESTURLPROP=`grep ^aai.server.url.base= $prop_file |cut -d'=' -f2 |tr -d "\015"` +if [ -z $RESTURLPROP ]; then + echo "Property [aai.server.url.base] not found in file $prop_file" + MISSING_PROP=true +else + RESTURL=`echo $RESTURLPROP | sed -e "s#aai/#$BASEPATH/#"` +fi + +if [ ! -z "$1" ]; then + if [[ $1 == *"https"* ]]; then + RESTURL=$1 + fi +fi + +USEBASICAUTH=false +BASICENABLE=`grep ^aai.tools.enableBasicAuth $prop_file |cut -d'=' -f2 |tr -d "\015"` +if [ -z $BASICENABLE ]; then + USEBASICAUTH=false +else + USEBASICAUTH=true + CURLUSER=`grep ^aai.tools.username $prop_file |cut -d'=' -f2 |tr -d "\015"` + if [ -z $CURLUSER ]; then + echo "Property [aai.tools.username] not found in file $prop_file" + MISSING_PROP=true + fi + CURLPASSWORD=`grep ^aai.tools.password $prop_file |cut -d'=' -f2 |tr -d "\015"` + if [ -z $CURLPASSWORD ]; then + echo "Property [aai.tools.password] not found in file $prop_file" + MISSING_PROP=true + fi +fi + +fname=$JSONFILE +if [ -f /tmp/$(basename $JSONFILE) ]; then + fname=/tmp/$(basename $JSONFILE) +elif [ ! -f $JSONFILE ]; then + echo "The file $JSONFILE does not exist" + exit -1 +fi + +if [ $MISSING_PROP = false ]; then + if [ $USEBASICAUTH = false ]; then + AUTHSTRING="--cert $PROJECT_HOME/resources/etc/auth/aaiClientPublicCert.pem --key $PROJECT_HOME/resources/etc/auth/aaiClientPrivateKey.pem" + else + AUTHSTRING="-u $CURLUSER:$CURLPASSWORD" + fi + curl --request PUT -k $AUTHSTRING -H "X-FromAppId: $XFROMAPPID" -H "X-TransactionId: $XTRANSID" -H "Accept: application/json" -H "Content-Type: application/json" -T $fname $RESTURL$RESOURCE | jq '.' > $RESULTPATH + RC=$? + +else + echo "usage: $0 <base-path>" + RC=-1 +fi +if [ "a$JSONFILE" = "a$TEMPFILE" ]; then + rm $TEMPFILE +fi +echo `date` " Done $0, exit code is $RC, returning result in $RESULTPATH" +exit $RC
\ No newline at end of file diff --git a/src/main/scripts/resend-dmaap-events.sh b/src/main/scripts/resend-dmaap-events.sh index 2afa3a7..b8ff73f 100644 --- a/src/main/scripts/resend-dmaap-events.sh +++ b/src/main/scripts/resend-dmaap-events.sh @@ -33,17 +33,21 @@ # # -b, (required) <string> the base url for the dmaap server # -e, (required) <file> filename containing the missed events -# -l, (optional) indicating that the script should be run it debug mode +# -l, (optional) indicating that the script should be run in debug mode # it will not send the dmaap messages to dmaap server # but it will write to a file named resend_dmaap_server.out +# -x (optional) skip resource version check # -p, (required) <string> the password for the dmaap server # -s, (required) <file> containing the data snapshot graphson file to compare the resource versions against -# -u, (required) <string> the username for the dmaap server +# partial snapshots should be concatenated into a full snapshot file +# before running the script +# -u, (required) <string> the username for the dmaap server +# -t, (required) <string> the dmaap topic # # An example of how to use the tool: # Please use right credentials and right dmaap server in the cluster # -# ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT # # For each dmaap message in the example_events.txt, it will check # against graphson and try to send it to the dmaap server @@ -66,9 +70,11 @@ # For testing purposes, if you are trying to run this script and don't want to actually # send it to a dmaap server, then you can run either of the following: # -# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -t <other-dmaap-topic> # or -# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 +# ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t <other-dmaap-topic> +# or, to skip the resource-version check +# ./resend-dmaap-events.sh -l -x -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t <other-dmaap-topic> # # Following will output what would have been sent out based on checking the datasnapshot with example_events.txt # @@ -90,18 +96,22 @@ usage(){ echo; echo " -b, <string> the base url for the dmaap server"; echo " -e, <file> filename containing the missed events"; - echo " -l, (optional) indicating that the script should be run it debug mode" + echo " -l, (optional) indicating that the script should be run in debug mode" echo " it will not send the dmaap messages to dmaap server " echo " but it will write to a file named resend_dmaap_server.out" + echo " -x, (optional) indicating that the script will skip the resource-version check" echo " -p, <string> the password for the dmaap server"; echo " -s, <file> containing the data snapshot graphson file to compare the resource versions against"; + echo " partial snapshots should be concatenated into a full snapshot file"; + echo " before running the script"; echo " -u, <string> the username for the dmaap server"; + echo " -t, <string> the dmaap topic"; echo; echo; echo " An example of how to use the tool:"; echo " Please use right credentials and right dmaap server in the cluster"; echo; - echo " ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo " ./resend-dmaap-events.sh -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; echo; echo " For each dmaap message in the example_events.txt, it will check "; echo " against graphson and try to send it to the dmaap server"; @@ -126,7 +136,9 @@ usage(){ echo; echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545"; echo " or"; - echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905"; + echo " ./resend-dmaap-events.sh -l -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; + echo " or, to skip the resource-version check"; + echo " ./resend-dmaap-events.sh -l -x -e example_events.txt -s dataSnapshot.graphSON.201808091545 -u username -p example_pass -b https://localhost:3905 -t AAI-EVENT"; echo; echo " Following will output what would have been sent out based on checking the datasnapshot with example_events.txt"; echo; @@ -143,25 +155,25 @@ usage(){ # Checks if the argument of the string is greater than zero # Also check if the file actually exists validate(){ - local type_of_file=$1; + type_of_file=$1; if [ $# -eq 0 ]; then echo "Error expecting the validate method to have at least one argument indicating what type"; - exit -1; + exit 1; fi; shift; - local arg=$1; + arg=$1; if [ -z "$arg" ]; then echo "Error missing the expected argument for ${type_of_file}"; - exit -1; + exit 1; fi; if [ ! -f "$arg" ]; then echo "Error: file $arg cannot be found, please check the file again"; - exit -1; + exit 1; fi; } @@ -171,26 +183,31 @@ validate(){ # Otherwise it will return non zero to indicate that this method failed resource_version_matches_snapshot_file(){ - local snapshot_file=$1; - local entity_link=$2; - local resource_version=$3; - local action=$4; + snapshot_file=$1; + entity_link=$2; + resource_version=$3; + action=$4; + topic=$5; if [ -z ${resource_version} ]; then echo "Missing the parameter resource version to be passed"; - return -1; + return 1; fi # Modify the entity link passed to remove the /aai/v[0-9]+ - aai_uri=$(echo $entity_link | sed 's/\/aai\/v[0-9][0-9]*//g'); + if [ "${topic}" = "<other-dmaap-topic>" ]; then + aai_uri=$(echo $entity_link | sed 's/\/<other-base>\/v[0-9][0-9]*//g'); + else + aai_uri=$(echo $entity_link | sed 's/\/aai\/v[0-9][0-9]*//g'); + fi - local line=$(grep '"value":"'${aai_uri}'"' ${snapshot_file} 2> /dev/null); + line=$(grep '"value":"'${aai_uri}'"' ${snapshot_file} 2> /dev/null); if [ -z "${line}" ] ; then if [ "${action}" = "DELETE" ]; then return 0; else - return -1; + return 1; fi; fi; @@ -199,51 +216,26 @@ resource_version_matches_snapshot_file(){ if [ $cnt -eq 1 ]; then return 0; else - return -1; + return 1; fi; } -# From a array being passed, it will determine the smallest element -# and return the index of the smallest element -# If the array length is zero, then it will return -1 -retrieve_smallest_index(){ - - local elements=("${@}"); - - if [ ${#elements} -eq 0 ]; then - return -1; - fi; - - local smallest_element=${elements[0]}; - - local index=0; - local smallest_index=0; - - for element in ${elements[@]}; do - if [ $element -lt $smallest_element ]; then - smallest_index=${index}; - fi; - index=$((index+1)); - done; - - return ${smallest_index}; -} - # Send the dmaap event to the host based on # the line that was send to the function send_dmaap(){ - local local_mode=$1; - local line=$2; - local username=$3; - local password=$4; - local baseurl=$5; - local resp_code=0; + local_mode=$1; + line=$2; + username=$3; + password=$4; + baseurl=$5; + topic=$6; + resp_code=0; generated_file=$(uuidgen); - local json_file=/tmp/${generated_file}.json; - local curl_output=/tmp/${generated_file}.txt; + json_file=/tmp/${generated_file}.json; + curl_output=/tmp/${generated_file}.txt; echo ${line} > ${json_file}; > ${curl_output}; @@ -259,13 +251,13 @@ send_dmaap(){ -X POST \ -H "Content-Type: application/json" \ -d "@${json_file}" \ - "${baseurl}/events/AAI-EVENT"\ + "${baseurl}/events/${topic}"\ ); if [ "$response_code" -ne "200" ]; then echo -n "Response failure for dmaap message with id ${id}," >> ${resend_error_log}; echo " code: ${response_code} body: $(cat ${curl_output})" >> ${resend_error_log}; - resp_code=-1; + resp_code=1; fi; fi; @@ -303,25 +295,32 @@ main(){ usage; fi; - while getopts ":e:s:u:lp:b:h" opt; do + versioncheck=true + while getopts ":e:s:u:xlp:b:t:h" opt; do case ${opt} in l ) # Specify that the application will not send messages to dmaap but save it a file - local local_mode=true + local_mode=true ;; e ) # Specify the file for missed events - local missed_events_file=$OPTARG + missed_events_file=$OPTARG ;; s ) # Specify the file for snapshot - local snapshot_file=$OPTARG + snapshot_file=$OPTARG ;; u ) # Specify the username to dmaap - local username=$OPTARG + username=$OPTARG ;; p ) # Specify the password to dmaap - local password=$OPTARG + password=$OPTARG + ;; + t ) # Specify the dmaap topic + topic=$OPTARG + ;; + x ) # Specify whether to skip version check + versioncheck=false ;; b ) # Specify the baseurl to dmaap - local hostname=$OPTARG + hostname=$OPTARG ;; h ) usage; @@ -345,15 +344,25 @@ main(){ id=$(echo $dmaap_event | grep -o '"id":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); action=$(echo $dmaap_event | grep -o '"action":"[^"]*"' | cut -d":" -f2- | sed 's/"//g'); smallest_resource_version=$(echo $dmaap_event | jq -M '.' | grep 'resource-version' | sort | tail -1 | sed 's/[^0-9]//g'); - resource_version_matches_snapshot_file "${snapshot_file}" "${entity_link}" "${smallest_resource_version}" "${action}" && { - send_dmaap "${local_mode}" "$dmaap_event" "$username" "$password" "$hostname" && { + #smallest_resource_version=$(echo $dmaap_event | python -m json.tool | grep 'resource-version' | sort | tail -1 | sed 's/[^0-9]//g'); + match=0; + if [ "$versioncheck" = true ]; then + resource_version_matches_snapshot_file "${snapshot_file}" "${entity_link}" "${smallest_resource_version}" "${action}" "${topic}" && { + match=0; + } || { + match=1; + } + fi; + + if [ $match -eq 0 ]; then + send_dmaap "${local_mode}" "$dmaap_event" "$username" "$password" "$hostname" "$topic" && { echo "Dmaap Event with Id $id was sent"; } || { echo "Dmaap Event with Id $id was not sent due to dmaap error, please check logs"; } - } || { + else echo "Dmaap Event with Id $id not sent"; - } + fi; done < ${missed_events_file}; diff --git a/src/main/scripts/run_SendDeleteMigrationNotification.sh b/src/main/scripts/run_SendDeleteMigrationNotification.sh index ebd8677..0b2f3e1 100644 --- a/src/main/scripts/run_SendDeleteMigrationNotification.sh +++ b/src/main/scripts/run_SendDeleteMigrationNotification.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/bin/bash ### # ============LICENSE_START======================================================= diff --git a/src/main/scripts/run_SendMigrationNotification.sh b/src/main/scripts/run_SendMigrationNotification.sh index 4bcc0d9..994485e 100644 --- a/src/main/scripts/run_SendMigrationNotification.sh +++ b/src/main/scripts/run_SendMigrationNotification.sh @@ -1,4 +1,4 @@ -#!/bin/ksh +#!/bin/bash ### # ============LICENSE_START======================================================= @@ -43,7 +43,7 @@ fi INPUT_DIR_FOR_JAVA=${INPUT_PATH}/combined mkdir -p "$INPUT_DIR_FOR_JAVA" INPUT_FILE_FOR_JAVA=${INPUT_DIR_FOR_JAVA}/sorted_dmaap_files.txt -sort --numeric-sort -k 1 -t '_' $(find ${INPUT_PATH}/* -maxdepth 0 -type f) | awk -F '_' '{ print $2"_"$3; }' > $INPUT_FILE_FOR_JAVA +sort -g -k 1 -t '_' $(find ${INPUT_PATH}/* -maxdepth 0 -type f) | awk -F '_' '{ print $2"_"$3; }' > $INPUT_FILE_FOR_JAVA shift |