diff options
author | 2019-01-03 16:54:50 +0100 | |
---|---|---|
committer | 2019-01-03 16:54:50 +0100 | |
commit | 03e61247159837b566b0cfcc380606dc317fbf1a (patch) | |
tree | 731792f18e95258f5d87747c98c70c833e9eaea9 /bash/tools/creating_data | |
parent | 329320e637f5ab3bea558db0e13452ab9cfc7bab (diff) |
Fix whitespaces across the bash scripts
Change-Id: I8f3912f1b7f35aa889ce63592b30014d6433a82f
Issue-ID: OOM-1551
Signed-off-by: Petr Ospalý <p.ospaly@partner.samsung.com>
Diffstat (limited to 'bash/tools/creating_data')
17 files changed, 136 insertions, 5 deletions
diff --git a/bash/tools/creating_data/create_nginx_image/01create-image.sh b/bash/tools/creating_data/create_nginx_image/01create-image.sh index 5469d3e7..86d96cf2 100644 --- a/bash/tools/creating_data/create_nginx_image/01create-image.sh +++ b/bash/tools/creating_data/create_nginx_image/01create-image.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash + # COPYRIGHT NOTICE STARTS HERE # # Copyright 2018 © Samsung Electronics Co., Ltd. @@ -16,6 +17,8 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + + script_dir="$(dirname ${BASH_SOURCE[0]})" cd "$script_dir" docker build -t own_nginx . diff --git a/bash/tools/creating_data/create_nginx_image/Dockerfile b/bash/tools/creating_data/create_nginx_image/Dockerfile index aafa264c..412d4edb 100644 --- a/bash/tools/creating_data/create_nginx_image/Dockerfile +++ b/bash/tools/creating_data/create_nginx_image/Dockerfile @@ -15,8 +15,18 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + FROM nginx:alpine + RUN apk add --no-cache --update fcgiwrap git git-daemon spawn-fcgi + +#RUN mkdir /srv/http +#VOLUME ["/var/log/nginx"] + COPY gitconfig /etc/ +#COPY nginx.conf /etc/nginx/nginx.conf + CMD spawn-fcgi -M 666 -s /var/run/fcgiwrap.socket /usr/bin/fcgiwrap && \ nginx -g "daemon off;" + +#CMD ["spawn-fcgi","-M 666","-s /var/run/fcgiwrap.socket","/usr/sbin/fcgiwrap"] diff --git a/bash/tools/creating_data/download-bin-tools.sh b/bash/tools/creating_data/download-bin-tools.sh index 31f4beb8..b48cd2df 100644 --- a/bash/tools/creating_data/download-bin-tools.sh +++ b/bash/tools/creating_data/download-bin-tools.sh @@ -15,13 +15,17 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + outdir="$1" if [[ -z "$outdir" ]]; then echo "Missing arg outdir" exit 1 fi + + mkdir -p "$outdir" cd "$outdir" + download() { url="$1" url_file="${url%%\?*}" @@ -29,13 +33,19 @@ download() { echo "Downloading $url" curl --retry 5 -y 10 -Y 10 --location "$url" -o "$file" } + download "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64" mv ./jq-linux64 ./jq + download "https://storage.googleapis.com/kubernetes-release/release/v1.8.10/bin/linux/amd64/kubectl" + download "https://storage.googleapis.com/kubernetes-helm/helm-v2.8.2-linux-amd64.tar.gz" tar -xf ./helm-v2.8.2-linux-amd64.tar.gz linux-amd64/helm -O > helm rm ./helm-v2.8.2-linux-amd64.tar.gz + download "https://github.com/rancher/cli/releases/download/v0.6.7/rancher-linux-amd64-v0.6.7.tar.gz" tar -xf ./rancher-linux-amd64-v0.6.7.tar.gz ./rancher-v0.6.7/rancher -O > rancher rm ./rancher-linux-amd64-v0.6.7.tar.gz -chmod a+x ./helm ./jq ./kubectl ./rancher
\ No newline at end of file + + +chmod a+x ./helm ./jq ./kubectl ./rancher diff --git a/bash/tools/creating_data/download-docker-images.sh b/bash/tools/creating_data/download-docker-images.sh index d0115547..121cd5af 100644 --- a/bash/tools/creating_data/download-docker-images.sh +++ b/bash/tools/creating_data/download-docker-images.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash + # COPYRIGHT NOTICE STARTS HERE # # Copyright 2018 © Samsung Electronics Co., Ltd. @@ -16,6 +17,8 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + + # boilerplate RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then @@ -23,17 +26,22 @@ if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh fi + SRC_IMAGE_LIST=$1 if [[ -z "$SRC_IMAGE_LIST" ]]; then SRC_IMAGE_LIST="docker_image_list.txt" fi + echo "Download all images" + lines=$(cat $SRC_IMAGE_LIST | wc -l) line=1 while read -r image; do echo "== pkg #$line of $lines ==" + name=$(echo $image|awk '{print $1}') digest=$(echo $image|awk '{print $2}') + echo "$name digest:$digest" if [[ "$digest" == "<none>" ]]; then retry docker -l error pull "$name" @@ -41,4 +49,5 @@ while read -r image; do retry docker -l error pull "$name" fi line=$((line+1)) -done < "$SRC_IMAGE_LIST"
\ No newline at end of file + +done < "$SRC_IMAGE_LIST" diff --git a/bash/tools/creating_data/download-git-repos.sh b/bash/tools/creating_data/download-git-repos.sh index 4cd936d0..9d651d93 100644 --- a/bash/tools/creating_data/download-git-repos.sh +++ b/bash/tools/creating_data/download-git-repos.sh @@ -15,17 +15,22 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + lists_dir="$1" + if [[ -z "$lists_dir" ]]; then echo "Missing argument for lists_dir" exit 1 fi + outdir="$2" if [[ -z "$outdir" ]]; then outdir="./git-repo" fi + mkdir -p "$outdir" cd "$outdir" # NOTE: will be better to use sh extension? sh $lists_dir/git_manual_list -sh $lists_dir/git_repos_list
\ No newline at end of file +sh $lists_dir/git_repos_list + diff --git a/bash/tools/creating_data/download-http-files.sh b/bash/tools/creating_data/download-http-files.sh index 267d7952..06f41354 100644 --- a/bash/tools/creating_data/download-http-files.sh +++ b/bash/tools/creating_data/download-http-files.sh @@ -15,20 +15,25 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + LIST_FILE="$1" if [[ -z "$LIST_FILE" ]]; then echo "Missing list file" exit 1 fi + outdir="$2" if [[ -z "$outdir" ]]; then echo "Missing output directory" exit 1 fi + lines=$(cat "$LIST_FILE" | wc -l) cnt=1 + # create output dir if not exists mkdir -p "$outdir" + while read -r line; do # www.springframework.org/schema/tool/spring-tool-4.3.xsd file="${line%%\?*}" @@ -39,4 +44,5 @@ while read -r line; do # drop below 10b/10s curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null cnt=$((cnt+1)) -done < "$LIST_FILE"
\ No newline at end of file + +done < "$LIST_FILE" diff --git a/bash/tools/creating_data/download-npm-pkgs.sh b/bash/tools/creating_data/download-npm-pkgs.sh index 00737f2c..9e8847c9 100644 --- a/bash/tools/creating_data/download-npm-pkgs.sh +++ b/bash/tools/creating_data/download-npm-pkgs.sh @@ -15,15 +15,19 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + LIST_FILE="$1" + if [[ -z "$LIST_FILE" ]]; then LIST_FILE="all_npm_list.txt" fi + outdir="$2" if [[ -z "$outdir" ]]; then echo "Missing arg outdir" exit 1 fi + mkdir -p "$outdir" cd "$outdir" lines=$(cat "$LIST_FILE" | wc -l) @@ -33,4 +37,5 @@ while read -r line; do # yallist@2.1.2 npm pack $line cnt=$((cnt+1)) -done < "$LIST_FILE"
\ No newline at end of file + +done < "$LIST_FILE" diff --git a/bash/tools/creating_data/download-oom.sh b/bash/tools/creating_data/download-oom.sh index f0b4152f..010eeec1 100644 --- a/bash/tools/creating_data/download-oom.sh +++ b/bash/tools/creating_data/download-oom.sh @@ -15,6 +15,7 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + outdir="$1" if [[ -z "$outdir" ]]; then echo "Missing output dir" @@ -25,9 +26,12 @@ if [[ -z "$patch_file" ]]; then echo "Missing patch file" exit 1 fi + cd "$outdir" git clone https://github.com/onap/oom.git cd oom echo "Checkout base commit which will be patched" git checkout -b patched_beijing bf47d706fc8b94fd1232960e90329a9a532c6a7b + patch -p1 < "$patch_file" + diff --git a/bash/tools/creating_data/download-pip.sh b/bash/tools/creating_data/download-pip.sh index 1121ffd5..dea60b52 100644 --- a/bash/tools/creating_data/download-pip.sh +++ b/bash/tools/creating_data/download-pip.sh @@ -15,24 +15,31 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + LIST_FILE="$1" if [[ -z "$LIST_FILE" ]]; then echo "Missing list file" exit 1 fi LIST_FILE=$(readlink -f "$LIST_FILE") + + outdir="$2" if [[ -z "$outdir" ]]; then echo "Missing output directory" exit 1 fi + lines=$(cat "$LIST_FILE" | wc -l) cnt=1 + # create output dir if not exists mkdir -p "$outdir" + cd "$outdir" while read -r line; do echo "Downloading $cnt / $lines: $line" pip download $line cnt=$((cnt+1)) + done < "$LIST_FILE" diff --git a/bash/tools/creating_data/download-pkg.sh b/bash/tools/creating_data/download-pkg.sh index 2a5290ca..03bfe07f 100644 --- a/bash/tools/creating_data/download-pkg.sh +++ b/bash/tools/creating_data/download-pkg.sh @@ -15,11 +15,13 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + outdir="$1" if [[ -z "$outdir" ]]; then echo "Missing output dir" exit 1 fi + # if onap.repo does not exists create it mkdir -p $outdir if [ ! -f "$outdir/onap.repo" ]; then @@ -31,8 +33,10 @@ enabled=1 gpgcheck=0 EOF fi + # this exact docker version is required by ONAP/beijing # it should be available in centos docker repo yumdownloader --resolve --destdir="$outdir" docker-ce-17.03.2.ce libtool-ltdl docker-ce-selinux + yumdownloader --resolve --destdir="$outdir" dnsmasq icewm firefox tigervnc-server perl* createrepo "$outdir" diff --git a/bash/tools/creating_data/load-docker-images.sh b/bash/tools/creating_data/load-docker-images.sh index e97fc6b9..685bed42 100644 --- a/bash/tools/creating_data/load-docker-images.sh +++ b/bash/tools/creating_data/load-docker-images.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash + # COPYRIGHT NOTICE STARTS HERE # # Copyright 2018 © Samsung Electronics Co., Ltd. @@ -16,6 +17,8 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + + # boilerplate RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then @@ -23,21 +26,26 @@ if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh fi + CLEAN=false + if [ -z "$NEXUS_HOST" ]; then echo "Independent run for inserting of additional docker images" CLEAN=true mv ~/.docker/config.json ~/.docker/config.json_backup 2>/dev/null source "$LOCAL_PATH/docker-login.sh" fi + IMG_DIR="$1" if [[ -z "$IMG_DIR" ]]; then IMG_DIR="$(pwd)/images" fi + if [[ ! -d "${IMG_DIR}" ]]; then echo "No ${IMG_DIR} to load images" exit 0 fi + load_image() { local image="$1" echo "[DEBUG] load ${image}" @@ -49,14 +57,17 @@ load_image() { # delete pushed image from docker retry docker rmi "$name" } + IMAGES=$(find ${IMG_DIR} -name "*.tar" -type f) lines=$(echo ${IMAGES} | wc -l) line=1 for image in ${IMAGES}; do echo "== pkg #$line of $lines ==" load_image "$image" + line=$((line+1)) done + if [ "$CLEAN" = true ]; then # onap is using different credentials for docker login which can be conflicted # with ours so better to clean this-up diff --git a/bash/tools/creating_data/make-docker-images-list.sh b/bash/tools/creating_data/make-docker-images-list.sh index 61b236f4..b925d91d 100644 --- a/bash/tools/creating_data/make-docker-images-list.sh +++ b/bash/tools/creating_data/make-docker-images-list.sh @@ -15,6 +15,9 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + readonly IMG_LIST="/tmp/docker_image_list.txt" + echo "gathering images" docker images --digests | grep -v 'IMAGE ID'|awk '{printf("%s:%s %s\n", $1, $2, $3)}' > $IMG_LIST + diff --git a/bash/tools/creating_data/make-git-http-list.sh b/bash/tools/creating_data/make-git-http-list.sh index 1e97d337..a724d6f0 100644 --- a/bash/tools/creating_data/make-git-http-list.sh +++ b/bash/tools/creating_data/make-git-http-list.sh @@ -1,3 +1,5 @@ +# COPYRIGHT NOTICE STARTS HERE +# # Copyright 2018 © Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,27 +15,38 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + if [[ -z "$LISTS_DIR" ]]; then LISTS_DIR=. echo "Using default output directory ." fi + OOM_PATH="$1" + if [[ -z "$OOM_PATH" ]]; then echo "Missing oom path" exit 1 fi + + GOUTPUT="$LISTS_DIR/git_repos_list" FOUTPUT="$LISTS_DIR/fetch_list.txt" + trim_last() { echo "${@:1:$#-1}"; } + TMP='/tmp/git_tmp_list' :> $TMP + :> $FOUTPUT + echo "Gathering git repos and list possible html data" + while read -r chart; do out="$(helm template $(dirname "$chart") 2>/dev/null)" gitcmds=$(echo "$out" | grep 'git clone') + if [[ -n "$gitcmds" ]] ; then while read gitcmd; do gitcmd=$(trim_last $gitcmd) @@ -43,6 +56,7 @@ while read -r chart; do echo "$full" >> $TMP done <<< "$gitcmds" fi + fetchcmds=$(echo "$out" | grep 'wget \|curl ' | grep -v 'HEALTH_CHECK_ENDPOINT\|PUT\|POST' ) if [[ -n "$fetchcmds" ]] ; then while read fetchcmd; do @@ -51,5 +65,10 @@ while read -r chart; do echo "$fetchcmd" >> $FOUTPUT done <<< "$fetchcmds" fi + + done <<< "$(find $OOM_PATH -name Chart.yaml)" + sort $TMP | uniq > $GOUTPUT + + diff --git a/bash/tools/creating_data/make-npm-list.sh b/bash/tools/creating_data/make-npm-list.sh index dd3be670..5901d54d 100644 --- a/bash/tools/creating_data/make-npm-list.sh +++ b/bash/tools/creating_data/make-npm-list.sh @@ -1,3 +1,5 @@ +# COPYRIGHT NOTICE STARTS HERE +# # Copyright 2018 © Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,9 +15,13 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + FIND_PATH="/var/lib/docker/aufs/mnt/" + LIST_FILE="/tmp/npm.list" + : > $LIST_FILE + echo "gathering npm packages from $FIND_PATH" while read -r line; do # yallist/2.1.2/package/package.json @@ -24,4 +30,6 @@ while read -r line; do # tag=$(echo $image|awk '{print $2}') # save_image "$name:$tag" echo "$name@$ver" >> $LIST_FILE + done <<< "$(find $FIND_PATH -path "*/.npm/*/package.json" | sed 's#^.*\.npm/\(.*\)$#\1#' | sort | uniq)" + diff --git a/bash/tools/creating_data/remote-list-gathering.sh b/bash/tools/creating_data/remote-list-gathering.sh index 245e1eec..73fb164a 100644 --- a/bash/tools/creating_data/remote-list-gathering.sh +++ b/bash/tools/creating_data/remote-list-gathering.sh @@ -1,3 +1,5 @@ +# COPYRIGHT NOTICE STARTS HERE +# # Copyright 2018 © Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +15,8 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + + if [[ -z "$ONAP_SERVERS" ]]; then # ONAP_SERVERS="oom-beijing-postRC2-master oom-beijing-postRC2-compute1 oom-beijing-postRC2-compute2" echo "Missing environment ONAP_SERVERS" @@ -22,15 +26,20 @@ if [[ -z "$LISTS_DIR" ]]; then LISTS_DIR=. echo "Using default output directory ." fi + echo "Using onap servers: $ONAP_SERVERS" + cd $(dirname $0) + for server in $ONAP_SERVERS; do + echo "=================================================" echo "Gathering docker images list from server: $server" echo "=================================================" scp ./make-docker-images-list.sh $server:/tmp/ ssh $server '/tmp/make-docker-images-list.sh;rm /tmp/make-docker-images-list.sh' scp $server:/tmp/docker_image_list.txt $LISTS_DIR/docker_image_list-${server}.txt + echo "=================================================" echo "Gathering NPM packages list from server: $server" echo "=================================================" @@ -38,5 +47,7 @@ for server in $ONAP_SERVERS; do ssh $server '/tmp/make-npm-list.sh; rm /tmp/make-npm-list.sh' scp $server:/tmp/npm.list $LISTS_DIR/npm_list-${server}.txt done + cat "$LISTS_DIR"/docker_image_list-*.txt | grep -v sonatype/nexus3 | grep -v own_nginx | sort | uniq > "$LISTS_DIR/docker_image_list.txt" cat "$LISTS_DIR"/npm_list-*.txt | grep -v '^@$' | sort | uniq > "$LISTS_DIR/npm_list.txt" + diff --git a/bash/tools/creating_data/save-docker-images.sh b/bash/tools/creating_data/save-docker-images.sh index 98facb0c..f4a5d3c7 100644 --- a/bash/tools/creating_data/save-docker-images.sh +++ b/bash/tools/creating_data/save-docker-images.sh @@ -1,4 +1,5 @@ #! /usr/bin/env bash + # COPYRIGHT NOTICE STARTS HERE # # Copyright 2018 © Samsung Electronics Co., Ltd. @@ -16,6 +17,8 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + + # boilerplate RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then @@ -23,19 +26,24 @@ if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh fi + IMG_DIR="$1" + if [[ -z "$IMG_DIR" ]]; then IMG_DIR="./images" fi + echo "Creating ${IMG_DIR}" if [[ ! -d "${IMG_DIR}" ]]; then mkdir -p "${IMG_DIR}" fi + save_image() { local name_tag=$1 echo "$name_tag" local img_name=$(echo "${name_tag}" | tr /: __) local img_path="${IMG_DIR}/${img_name}.tar" + if [[ ! -f "${img_path}" ]] ; then echo "[DEBUG] save ${name_tag} to ${img_path}" echo "${name_tag}" >> $IMG_DIR/_image_list.txt @@ -44,13 +52,16 @@ save_image() { echo "[DEBUG] ${name_tag} already saved" fi } + echo "Save all images" line=1 lines=$(docker images|grep -v 'IMAGE ID'|wc -l) while read -r image; do echo "== pkg #$line of $lines ==" + name=$(echo $image|awk '{print $1}') tag=$(echo $image|awk '{print $2}') save_image "$name:$tag" line=$((line+1)) + done <<< "$(docker images|grep -v 'IMAGE ID'|awk '{printf("%s %s\n", $1, $2)}'|column -t)" diff --git a/bash/tools/creating_data/upload-maven-files.sh b/bash/tools/creating_data/upload-maven-files.sh index bd071fbc..07d6e7ab 100644 --- a/bash/tools/creating_data/upload-maven-files.sh +++ b/bash/tools/creating_data/upload-maven-files.sh @@ -15,6 +15,7 @@ # limitations under the License. # # COPYRIGHT NOTICE ENDS HERE + DATA_DIR="$1" if [[ -z "$DATA_DIR" ]]; then # needs for example: /root/onap-offline-installer/http @@ -22,10 +23,13 @@ if [[ -z "$DATA_DIR" ]]; then echo "Usage: $0 <path to http dir> <name of server> [<next server>...]" exit 1 fi + shift cd "$DATA_DIR" + for server in $*; do echo "Uploading to server: $server" + lines=$(find $server/ -type f | wc -l) count=1 while read -r url; do @@ -34,5 +38,6 @@ for server in $*; do path="$url" echo "Uploading file: $url" curl -u admin:admin123 --upload-file $path http://$url + done <<< "$(find $server/ -type f)" done |