Merge "Pushing script for http components download."

1 files changed, 42 insertions, 0 deletions

diff --git a/bash/tools/creating_data/download-http-files.sh b/bash/tools/creating_data/download-http-files.sh
new file mode 100644
index 00000000..267d7952
--- /dev/null
+++ b/bash/tools/creating_data/download-http-files.sh
@@ -0,0 +1,42 @@
+#   COPYRIGHT NOTICE STARTS HERE
+#
+#   Copyright 2018 © Samsung Electronics Co., Ltd.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#   COPYRIGHT NOTICE ENDS HERE
+LIST_FILE="$1"
+if [[ -z "$LIST_FILE" ]]; then
+    echo "Missing list file"
+    exit 1
+fi
+outdir="$2"
+if [[ -z "$outdir" ]]; then
+    echo "Missing output directory"
+    exit 1
+fi
+lines=$(cat "$LIST_FILE" | wc -l)
+cnt=1
+# create output dir if not exists
+mkdir -p "$outdir"
+while read -r line; do
+    # www.springframework.org/schema/tool/spring-tool-4.3.xsd
+    file="${line%%\?*}"
+    echo "Downloading $cnt / $lines: $file"
+    fdir=$(dirname "$file")
+    mkdir -p $outdir/$fdir
+    # following curl params are ensurring 5 reties and cut-off if connectivity will
+    # drop below 10b/10s
+    curl --retry 5 -y 10 -Y 10 --location  "$line" -o "$outdir/$file" &>/dev/null
+    cnt=$((cnt+1))
+done < "$LIST_FILE"
\ No newline at end of file