Fix download scripts to handle empty lines

Fixing download and saving scripts to be able to ignore empty or 00D
ending lines in lists

Issue-ID: OOM-1818

Change-Id: I55ef1dfbea628f8c1d4b19745536e629a2dbe0c9
Signed-off-by: Tomáš Levora <t.levora@partner.samsung.com>

7 files changed, 52 insertions, 57 deletions

diff --git a/build/common-functions.sh b/build/common-functions.sh
index e39c4778..04ea2017 100755
--- a/build/common-functions.sh
+++ b/build/common-functions.sh
@@ -90,3 +90,9 @@ retry() {
     done
 }
 
+clean_list() {
+    sed -e 's/\s*#.*$//' \
+        -e '/^\s*$/d' ${1} |
+    tr -d '\r' |
+    awk '{ print $1 }'
+}
diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh
index 121cd5af..c0a0bed1 100755
--- a/build/creating_data/download-docker-images.sh
+++ b/build/creating_data/download-docker-images.sh
@@ -19,35 +19,21 @@
 #   COPYRIGHT NOTICE ENDS HERE
 
 
-# boilerplate
-RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh'
-if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then
-    SCRIPT_DIR=$(dirname "${0}")
-    LOCAL_PATH=$(readlink -f "$SCRIPT_DIR")
-    . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh
-fi
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
 
-SRC_IMAGE_LIST=$1
-if [[ -z "$SRC_IMAGE_LIST" ]]; then
-    SRC_IMAGE_LIST="docker_image_list.txt"
+LIST_FILE="${1}"
+if [[ -z "$LIST_FILE" ]]; then
+    LIST_FILE="docker_image_list.txt"
 fi
 
 echo "Download all images"
 
-lines=$(cat $SRC_IMAGE_LIST | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
 line=1
-while read -r image; do
+for image in $(clean_list "$LIST_FILE"); do
     echo "== pkg #$line of $lines =="
-
-    name=$(echo $image|awk '{print $1}')
-    digest=$(echo $image|awk '{print $2}')
-
-    echo "$name digest:$digest"
-    if [[ "$digest" == "<none>" ]]; then
-        retry docker -l error pull "$name"
-    else
-        retry docker -l error pull "$name"
-    fi
+    echo "$image"
+    retry docker -l error pull "$image"
     line=$((line+1))
-
-done < "$SRC_IMAGE_LIST"
+done
diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh
index 89e2026c..f687fda1 100755
--- a/build/creating_data/download-files.sh
+++ b/build/creating_data/download-files.sh
@@ -16,7 +16,11 @@
 #
 #   COPYRIGHT NOTICE ENDS HERE
 
-LIST_FILE="$1"
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
 if [[ -z "$LIST_FILE" ]]; then
     echo "Missing list file"
     exit 1
@@ -28,13 +32,13 @@ if [[ -z "$outdir" ]]; then
     exit 1
 fi
 
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
 cnt=1
 
 # create output dir if not exists
 mkdir -p "$outdir"
 
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
     # www.springframework.org/schema/tool/spring-tool-4.3.xsd
     file="${line%%\?*}"
     filename=$(basename "$file")
@@ -43,5 +47,4 @@ while read -r line; do
     # drop below 10b/10s
     curl --retry 5 -y 10 -Y 10 --location  "$line" -o "$outdir/$filename" &>/dev/null
     cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh
index 06f41354..1144c66e 100755
--- a/build/creating_data/download-http-files.sh
+++ b/build/creating_data/download-http-files.sh
@@ -16,7 +16,11 @@
 #
 #   COPYRIGHT NOTICE ENDS HERE
 
-LIST_FILE="$1"
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
 if [[ -z "$LIST_FILE" ]]; then
     echo "Missing list file"
     exit 1
@@ -28,13 +32,13 @@ if [[ -z "$outdir" ]]; then
     exit 1
 fi
 
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
 cnt=1
 
 # create output dir if not exists
 mkdir -p "$outdir"
 
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
     # www.springframework.org/schema/tool/spring-tool-4.3.xsd
     file="${line%%\?*}"
     echo "Downloading $cnt / $lines: $file"
@@ -44,5 +48,4 @@ while read -r line; do
     # drop below 10b/10s
     curl --retry 5 -y 10 -Y 10 --location  "$line" -o "$outdir/$file" &>/dev/null
     cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh
index 9e8847c9..191dd5df 100755
--- a/build/creating_data/download-npm-pkgs.sh
+++ b/build/creating_data/download-npm-pkgs.sh
@@ -16,7 +16,10 @@
 #
 #   COPYRIGHT NOTICE ENDS HERE
 
-LIST_FILE="$1"
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
+LIST_FILE="${1}"
 
 if [[ -z "$LIST_FILE" ]]; then
     LIST_FILE="all_npm_list.txt"
@@ -30,12 +33,10 @@ fi
 
 mkdir -p "$outdir"
 cd "$outdir"
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
 cnt=1
-while read -r line; do
+for line in $(clean_list "$LIST_FILE"); do
     echo "== pkg #$cnt of $lines =="
-    # yallist@2.1.2
     npm pack $line
     cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
\ No newline at end of file
diff --git a/build/creating_data/download-pip.sh b/build/creating_data/download-pip.sh
index dea60b52..fd4a4363 100755
--- a/build/creating_data/download-pip.sh
+++ b/build/creating_data/download-pip.sh
@@ -16,6 +16,10 @@
 #
 #   COPYRIGHT NOTICE ENDS HERE
 
+
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
+
 LIST_FILE="$1"
 if [[ -z "$LIST_FILE" ]]; then
     echo "Missing list file"
@@ -30,16 +34,15 @@ if [[ -z "$outdir" ]]; then
     exit 1
 fi
 
-lines=$(cat "$LIST_FILE" | wc -l)
+lines=$(clean_list "$LIST_FILE" | wc -l)
 cnt=1
 
 # create output dir if not exists
 mkdir -p "$outdir"
 
 cd "$outdir"
-while read -r line; do
+for line in $(clean_list "$LIST_FILE)"; do
     echo "Downloading $cnt / $lines: $line"
     pip download $line
     cnt=$((cnt+1))
-
-done < "$LIST_FILE"
+done
diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh
index 4c764556..0a72d15b 100755
--- a/build/creating_data/save-docker-images.sh
+++ b/build/creating_data/save-docker-images.sh
@@ -19,15 +19,10 @@
 #   COPYRIGHT NOTICE ENDS HERE
 
 
-# boilerplate
-RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh'
-if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then
-    SCRIPT_DIR=$(dirname "${0}")
-    LOCAL_PATH=$(readlink -f "$SCRIPT_DIR")
-    . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh
-fi
+# Load common-functions library
+. $(dirname ${0})/../common-functions.sh
 
-LIST="${1}"
+LIST_FILE="${1}"
 IMG_DIR="${2}"
 
 if [[ -z "$IMG_DIR" ]]; then
@@ -56,11 +51,9 @@ save_image() {
 
 echo "Save all images"
 line=1
-lines=$(wc -l ${LIST})
-while read -r image; do
+lines=$(clean_list "$LIST_FILE" | wc -l)
+for image in $(clean_list "$LIST_FILE"); do
     echo "== pkg #$line of $lines =="
-
     save_image "${image}"
     line=$((line+1))
-
-done < "${LIST}"
+done
\ No newline at end of file