summaryrefslogtreecommitdiffstats
path: root/build/creating_data/download-http-files.sh
diff options
context:
space:
mode:
Diffstat (limited to 'build/creating_data/download-http-files.sh')
-rwxr-xr-xbuild/creating_data/download-http-files.sh48
1 files changed, 48 insertions, 0 deletions
diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh
new file mode 100755
index 00000000..06f41354
--- /dev/null
+++ b/build/creating_data/download-http-files.sh
@@ -0,0 +1,48 @@
+# COPYRIGHT NOTICE STARTS HERE
+#
+# Copyright 2018 © Samsung Electronics Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# COPYRIGHT NOTICE ENDS HERE
+
+LIST_FILE="$1"
+if [[ -z "$LIST_FILE" ]]; then
+ echo "Missing list file"
+ exit 1
+fi
+
+outdir="$2"
+if [[ -z "$outdir" ]]; then
+ echo "Missing output directory"
+ exit 1
+fi
+
+lines=$(cat "$LIST_FILE" | wc -l)
+cnt=1
+
+# create output dir if not exists
+mkdir -p "$outdir"
+
+while read -r line; do
+ # www.springframework.org/schema/tool/spring-tool-4.3.xsd
+ file="${line%%\?*}"
+ echo "Downloading $cnt / $lines: $file"
+ fdir=$(dirname "$file")
+ mkdir -p $outdir/$fdir
+ # following curl params are ensurring 5 reties and cut-off if connectivity will
+ # drop below 10b/10s
+ curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null
+ cnt=$((cnt+1))
+
+done < "$LIST_FILE"