Merge changes I6421b55d,Icdb55918

* changes:
  Wrapping-up download refactorisation
  Adding rpm data list

12 files changed, 21 insertions, 471 deletions

diff --git a/build/creating_data/create-rhel-repo.sh b/build/creating_data/create-rhel-repo.sh
deleted file mode 100755
index 43709a7e..00000000
--- a/build/creating_data/create-rhel-repo.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018-2019 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-OUTDIR="${1}"
-if [[ -z "${OUTDIR}" ]]; then
-    echo "Missing output dir"
-    exit 1
-fi
-
-# if onap.repo does not exists create it
-mkdir -p "${OUTDIR}"
-if [ ! -f "${OUTDIR}/onap.repo" ]; then
-   cat > "${OUTDIR}/onap.repo" <<EOF
-[ONAP]
-name=Offline ONAP repository
-baseurl=PATH
-enabled=1
-gpgcheck=0
-EOF
-fi
-
-# this exact docker version is required by ONAP/beijing
-# it should be available in centos docker repo
-yumdownloader --resolve --destdir="${OUTDIR}" docker-ce-18.09.5 container-selinux docker-ce-cli \
-containerd.io nfs-utils python-jsonpointer python-docker-py python-docker-pycreds python-ipaddress \
-python-websocket-client
-
-createrepo "${OUTDIR}"
-
-exit 0
\ No newline at end of file
diff --git a/build/creating_data/create-ubuntu-repo.sh b/build/creating_data/create-ubuntu-repo.sh
deleted file mode 100755
index ac7de65c..00000000
--- a/build/creating_data/create-ubuntu-repo.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-OUTDIR="${1}"
-if [[ -z "${OUTDIR}" ]]; then
-    echo "Missing output dir"
-    exit 1
-fi
-
-
-# create the package index
-dpkg-scanpackages -m "${OUTDIR}" > "${OUTDIR}/Packages"
-cat "${OUTDIR}/Packages" | gzip -9c > "${OUTDIR}/Packages.gz"
-
-# create the Release file
-echo 'deb [trusted=yes] http://repo.infra-server/ubuntu/xenial /' > "${OUTDIR}/onap.list"
-
-exit 0
diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh
deleted file mode 100755
index c0a0bed1..00000000
--- a/build/creating_data/download-docker-images.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /usr/bin/env bash
-
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-# Load common-functions library
-. $(dirname ${0})/../common-functions.sh
-
-LIST_FILE="${1}"
-if [[ -z "$LIST_FILE" ]]; then
-    LIST_FILE="docker_image_list.txt"
-fi
-
-echo "Download all images"
-
-lines=$(clean_list "$LIST_FILE" | wc -l)
-line=1
-for image in $(clean_list "$LIST_FILE"); do
-    echo "== pkg #$line of $lines =="
-    echo "$image"
-    retry docker -l error pull "$image"
-    line=$((line+1))
-done
diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh
deleted file mode 100755
index f687fda1..00000000
--- a/build/creating_data/download-files.sh
+++ /dev/null
@@ -1,50 +0,0 @@
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-# Load common-functions library
-. $(dirname ${0})/../common-functions.sh
-
-LIST_FILE="${1}"
-if [[ -z "$LIST_FILE" ]]; then
-    echo "Missing list file"
-    exit 1
-fi
-
-outdir="$2"
-if [[ -z "$outdir" ]]; then
-    echo "Missing output directory"
-    exit 1
-fi
-
-lines=$(clean_list "$LIST_FILE" | wc -l)
-cnt=1
-
-# create output dir if not exists
-mkdir -p "$outdir"
-
-for line in $(clean_list "$LIST_FILE"); do
-    # www.springframework.org/schema/tool/spring-tool-4.3.xsd
-    file="${line%%\?*}"
-    filename=$(basename "$file")
-    echo "Downloading $cnt / $lines: $file"
-    # following curl params are ensurring 5 reties and cut-off if connectivity will
-    # drop below 10b/10s
-    curl --retry 5 -y 10 -Y 10 --location  "$line" -o "$outdir/$filename" &>/dev/null
-    cnt=$((cnt+1))
-done
\ No newline at end of file
diff --git a/build/creating_data/download-git-repos.sh b/build/creating_data/download-git-repos.sh
deleted file mode 100755
index 7853a141..00000000
--- a/build/creating_data/download-git-repos.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#! /usr/bin/env bash
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-# fail fast
-set -e
-
-usage () {
-    echo "Usage:"
-    echo -e "./$(basename $0) <repository list> [destination directory]\n"
-    echo "Examples:"
-    echo "  ./$(basename $0) onap_3.0.x-git_repos.list ./git-repo"
-}
-
-LIST="${1}"
-
-if [[ -z "${LIST}" ]]; then
-    echo "Missing argument for repository list"
-    exit 1
-fi
-
-OUTDIR="${2}"
-if [[ -z "${OUTDIR}" ]]; then
-    OUTDIR="./git-repo"
-fi
-
-mkdir -p "${OUTDIR}"
-cd "${OUTDIR}"
-
-
-while IFS=" " read -r REPO BRANCH remainder
-do
-        if [[ -z "${BRANCH}" ]]; then
-                git clone https://${REPO} --bare ${REPO}
-        else
-                git clone -b ${BRANCH} --single-branch https://${REPO} --bare ${REPO}
-        fi
-done < <(awk '$1 ~ /^[^;#]/' ${LIST})
-
-
-exit 0
diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh
deleted file mode 100755
index 1144c66e..00000000
--- a/build/creating_data/download-http-files.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-# Load common-functions library
-. $(dirname ${0})/../common-functions.sh
-
-LIST_FILE="${1}"
-if [[ -z "$LIST_FILE" ]]; then
-    echo "Missing list file"
-    exit 1
-fi
-
-outdir="$2"
-if [[ -z "$outdir" ]]; then
-    echo "Missing output directory"
-    exit 1
-fi
-
-lines=$(clean_list "$LIST_FILE" | wc -l)
-cnt=1
-
-# create output dir if not exists
-mkdir -p "$outdir"
-
-for line in $(clean_list "$LIST_FILE"); do
-    # www.springframework.org/schema/tool/spring-tool-4.3.xsd
-    file="${line%%\?*}"
-    echo "Downloading $cnt / $lines: $file"
-    fdir=$(dirname "$file")
-    mkdir -p $outdir/$fdir
-    # following curl params are ensurring 5 reties and cut-off if connectivity will
-    # drop below 10b/10s
-    curl --retry 5 -y 10 -Y 10 --location  "$line" -o "$outdir/$file" &>/dev/null
-    cnt=$((cnt+1))
-done
\ No newline at end of file
diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh
deleted file mode 100755
index 191dd5df..00000000
--- a/build/creating_data/download-npm-pkgs.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-# Load common-functions library
-. $(dirname ${0})/../common-functions.sh
-
-LIST_FILE="${1}"
-
-if [[ -z "$LIST_FILE" ]]; then
-    LIST_FILE="all_npm_list.txt"
-fi
-
-outdir="$2"
-if [[ -z "$outdir" ]]; then
-    echo "Missing arg outdir"
-    exit 1
-fi
-
-mkdir -p "$outdir"
-cd "$outdir"
-lines=$(clean_list "$LIST_FILE" | wc -l)
-cnt=1
-for line in $(clean_list "$LIST_FILE"); do
-    echo "== pkg #$cnt of $lines =="
-    npm pack $line
-    cnt=$((cnt+1))
-done
\ No newline at end of file
diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh
deleted file mode 100755
index 0a72d15b..00000000
--- a/build/creating_data/save-docker-images.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#! /usr/bin/env bash
-
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-# Load common-functions library
-. $(dirname ${0})/../common-functions.sh
-
-LIST_FILE="${1}"
-IMG_DIR="${2}"
-
-if [[ -z "$IMG_DIR" ]]; then
-    IMG_DIR="./images"
-fi
-
-echo "Creating ${IMG_DIR}"
-if [[ ! -d "${IMG_DIR}" ]]; then
-    mkdir -p "${IMG_DIR}"
-fi
-
-save_image() {
-    local name_tag=$1
-    echo "$name_tag"
-    local img_name=$(echo "${name_tag}" | tr /: __)
-    local img_path="${IMG_DIR}/${img_name}.tar"
-
-    if [[ ! -f "${img_path}" ]] ; then
-        echo "[DEBUG] save ${name_tag} to ${img_path}"
-        echo "${name_tag}" >> $IMG_DIR/_image_list.txt
-        retry docker -l error save -o "${img_path}" ${name_tag}
-    else
-        echo "[DEBUG] ${name_tag} already saved"
-    fi
-}
-
-echo "Save all images"
-line=1
-lines=$(clean_list "$LIST_FILE" | wc -l)
-for image in $(clean_list "$LIST_FILE"); do
-    echo "== pkg #$line of $lines =="
-    save_image "${image}"
-    line=$((line+1))
-done
\ No newline at end of file
diff --git a/build/data_lists/onap_rpm.list b/build/data_lists/onap_rpm.list
new file mode 100644
index 00000000..4595d4b1
--- /dev/null
+++ b/build/data_lists/onap_rpm.list
@@ -0,0 +1,21 @@
+containerd.io-1.2.5-3.1.el7.x86_64
+container-selinux-1.12.5-14.el7.x86_64
+container-selinux-2.95-2.el7_6.noarch
+docker-ce-18.09.5-3.el7.x86_64
+docker-ce-cli-18.09.6-3.el7.x86_64
+gssproxy-0.7.0-21.el7.x86_64
+keyutils-1.5.8-3.el7.x86_64
+libbasicobjects-0.1.1-32.el7.x86_64
+libcollection-0.7.0-32.el7.x86_64
+libevent-2.0.21-4.el7.x86_64
+libini_config-1.3.1-32.el7.x86_64
+libnfsidmap-0.25-19.el7.x86_64
+libpath_utils-0.2.1-32.el7.x86_64
+libref_array-0.1.5-32.el7.x86_64
+libverto-libevent-0.2.5-4.el7.x86_64
+nfs-utils-1.3.0-0.61.el7.x86_64
+python-docker-py-1.10.6-9.el7_6.noarch
+python-docker-pycreds-0.3.0-9.el7_6.noarch
+python-ipaddress-1.0.16-2.el7.noarch
+python-jsonpointer-1.9-2.el7.noarch
+python-websocket-client-0.32.0-116.el7.noarch
diff --git a/build/creating_data/download-bin-tools.sh b/build/download/download-bin-tools.sh
index 327e210f..327e210f 100755
--- a/build/creating_data/download-bin-tools.sh
+++ b/build/download/download-bin-tools.sh
diff --git a/build/creating_data/download-pip.sh b/build/download/download-pip.sh
index 7ab4b0c8..7ab4b0c8 100755
--- a/build/creating_data/download-pip.sh
+++ b/build/download/download-pip.sh
diff --git a/build/download_offline_data_by_lists.sh b/build/download_offline_data_by_lists.sh
deleted file mode 100755
index b2afd172..00000000
--- a/build/download_offline_data_by_lists.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#! /usr/bin/env bash
-
-#   COPYRIGHT NOTICE STARTS HERE
-#
-#   Copyright 2018 © Samsung Electronics Co., Ltd.
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-#   COPYRIGHT NOTICE ENDS HERE
-
-
-# fail fast
-set -e
-
-usage () {
-    echo "Usage:"
-    echo -e "./$(basename $0) <project version>\n"
-    echo "onap_3.0.0 for casablanca                                (sign-off 30/11/2018)"
-    echo "onap_3.0.1 for casablanca maintenance release            (sign-off 10/12/2018)"
-    echo "onap_3.0.2 for latest casablanca with fixed certificates (sign-off 25/04/2019)"
-    echo ""
-    echo "Example:"
-    echo "  ./$(basename $0) onap_3.0.2"
-}
-
-# boilerplate
-RELATIVE_PATH=./ # relative path from this script to 'common-functions.sh'
-if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then
-    SCRIPT_DIR=$(dirname "${0}")
-    LOCAL_PATH=$(readlink -f "$SCRIPT_DIR")
-    . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh
-fi
-
-if [ "${1}" == "-h" ] || [ -z "${1}" ]; then
-    usage
-    exit 0
-else
-    TAG="${1}"
-fi
-
-CTOOLS="${LOCAL_PATH}/creating_data"
-LISTS_DIR="${LOCAL_PATH}/data_lists"
-DATA_DIR="${LOCAL_PATH}/../../resources"
-TOTAL=12
-CURR=1
-
-message info "Downloading started: $(date)"
-
-echo "[Step $((CURR++))/$TOTAL Download collected docker images]"
-$CTOOLS/download-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list"
-
-echo "[Step $((CURR++))/$TOTAL Download docker images for infra-server]"
-$CTOOLS/download-docker-images.sh "${LISTS_DIR}/infra_docker_images.list"
-
-echo "[Step $((CURR++))/$TOTAL Build own nginx image]"
-$CTOOLS/create_nginx_image/01create-image.sh "${DATA_DIR}/offline_data/docker_images_infra"
-
-echo "[Step $((CURR++))/$TOTAL Save docker images from docker cache to tarfiles]"
-$CTOOLS/save-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list" "${DATA_DIR}/offline_data/docker_images_for_nexus"
-
-echo "[Step $((CURR++))/$TOTAL Prepare infra related images to infra folder]"
-$CTOOLS/save-docker-images.sh "${LISTS_DIR}/infra_docker_images.list" "${DATA_DIR}/offline_data/docker_images_infra"
-
-echo "[Step $((CURR++))/$TOTAL Download git repos]"
-$CTOOLS/download-git-repos.sh "${LISTS_DIR}/onap_3.0.x-git_repos.list" "${DATA_DIR}/git-repo"
-
-echo "[Step $((CURR++))/$TOTAL Download http files]"
-$CTOOLS/download-http-files.sh "${LISTS_DIR}/onap_3.0.x-http_files.list" "${DATA_DIR}/http"
-
-echo "[Step $((CURR++))/$TOTAL Download npm pkgs]"
-$CTOOLS/download-npm-pkgs.sh "${LISTS_DIR}/onap_3.0.x-npm.list" "${DATA_DIR}/offline_data/npm_tar"
-
-echo "[Step $((CURR++))/$TOTAL Download bin tools]"
-$CTOOLS/download-bin-tools.sh "${DATA_DIR}/downloads"
-
-echo "[Step $((CURR++))/$TOTAL Create RHEL repository]"
-$CTOOLS/create-rhel-repo.sh "${DATA_DIR}/pkg/rhel"
-
-echo "[Step $((CURR++))/$TOTAL Download sdnc-ansible-server packages]"
-$CTOOLS/download-pip.sh "${LISTS_DIR}/onap_3.0.x-pip_packages.list" "${DATA_DIR}/offline_data/pypi"
-$CTOOLS/download-files.sh "${LISTS_DIR}/deb_packages.list" "${DATA_DIR}/pkg/ubuntu/xenial"
-
-echo "[Step $((CURR++))/$TOTAL Create APT repository]"
-$CTOOLS/create-ubuntu-repo.sh "${DATA_DIR}/pkg/ubuntu/xenial"
-
-message info "Downloading finished: $(date)"