diff options
Diffstat (limited to 'build')
26 files changed, 415 insertions, 702 deletions
diff --git a/build/build_nexus_blob.sh b/build/build_nexus_blob.sh index 09ed8969..f3edb482 100755 --- a/build/build_nexus_blob.sh +++ b/build/build_nexus_blob.sh @@ -20,7 +20,7 @@ ### This script prepares Nexus repositories data blobs for ONAP -## The script requires following dependencies are installed: nodejs, jq, docker +## The script requires following dependencies are installed: nodejs, jq, docker, twine ## All required resources are expected in the upper directory created during ## download procedure as DATA_DIR or in the directory given as --input-directory ## All lists used must be in project data_lists directory or in the directory given @@ -58,22 +58,18 @@ NEXUS_DATA_DIR="${DATA_DIR}/nexus_data" LISTS_DIR="${LOCAL_PATH}/data_lists" usage () { - echo " Example usage: build_nexus_blob.sh -t <tag> --input-directory </path/to/downloaded/files/dir> --output-directory + echo " Example usage: build_nexus_blob.sh --input-directory </path/to/downloaded/files/dir> --output-directory </path/to/output/dir> --resource-list-directory </path/to/dir/with/resource/list> - -t | --tag release tag, taken from available on git or placed by data generating script (mandatory) must fallow scheme onap_<semver> -i | --input-directory directory containing file needed to create nexus blob. The structure of this directory must organized as described in build guide -o | --output-directory - -rl | --resource-list-directory directory with files containing docker, pypi and rpm lists + -rl | --resource-list-directory directory with files containing docker, pypi and npm lists " exit 1 } while [ "$1" != "" ]; do case $1 in - -t | --tag ) shift - TAG=$1 - ;; -i | --input-directory ) shift DATA_DIR=$1 ;; @@ -90,22 +86,15 @@ while [ "$1" != "" ]; do shift done - -# exit if no tag given -if [ -z ${TAG} ]; then - usage - exit 1 -fi - # Setup directories with resources for docker, npm and pypi NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus" NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar" NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi" -# Setup specific resources list based on the tag provided -NXS_DOCKER_IMG_LIST="${LISTS_DIR}/${TAG}-docker_images.list" -NXS_NPM_LIST="${LISTS_DIR}/$(sed 's/.$/x/' <<< ${TAG})-npm.list" -NXS_PYPI_LIST="${LISTS_DIR}/$(sed 's/.$/x/' <<< ${TAG})-pip_packages.list" +# Setup specific resources lists +NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list" +NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list" +NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list" # Setup Nexus image used for build and install infra INFRA_LIST="${LISTS_DIR}/infra_docker_images.list" @@ -340,4 +329,3 @@ npm config set registry "https://registry.npmjs.org" echo "Nexus blob is built" exit 0 - diff --git a/build/creating_data/create-rhel-repo.sh b/build/creating_data/create-rhel-repo.sh deleted file mode 100755 index 43709a7e..00000000 --- a/build/creating_data/create-rhel-repo.sh +++ /dev/null @@ -1,45 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -OUTDIR="${1}" -if [[ -z "${OUTDIR}" ]]; then - echo "Missing output dir" - exit 1 -fi - -# if onap.repo does not exists create it -mkdir -p "${OUTDIR}" -if [ ! -f "${OUTDIR}/onap.repo" ]; then - cat > "${OUTDIR}/onap.repo" <<EOF -[ONAP] -name=Offline ONAP repository -baseurl=PATH -enabled=1 -gpgcheck=0 -EOF -fi - -# this exact docker version is required by ONAP/beijing -# it should be available in centos docker repo -yumdownloader --resolve --destdir="${OUTDIR}" docker-ce-18.09.5 container-selinux docker-ce-cli \ -containerd.io nfs-utils python-jsonpointer python-docker-py python-docker-pycreds python-ipaddress \ -python-websocket-client - -createrepo "${OUTDIR}" - -exit 0
\ No newline at end of file diff --git a/build/creating_data/create-ubuntu-repo.sh b/build/creating_data/create-ubuntu-repo.sh deleted file mode 100755 index ac7de65c..00000000 --- a/build/creating_data/create-ubuntu-repo.sh +++ /dev/null @@ -1,33 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -OUTDIR="${1}" -if [[ -z "${OUTDIR}" ]]; then - echo "Missing output dir" - exit 1 -fi - - -# create the package index -dpkg-scanpackages -m "${OUTDIR}" > "${OUTDIR}/Packages" -cat "${OUTDIR}/Packages" | gzip -9c > "${OUTDIR}/Packages.gz" - -# create the Release file -echo 'deb [trusted=yes] http://repo.infra-server/ubuntu/xenial /' > "${OUTDIR}/onap.list" - -exit 0 diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh index e13b9150..6761c328 100755 --- a/build/creating_data/docker-images-collector.sh +++ b/build/creating_data/docker-images-collector.sh @@ -30,9 +30,9 @@ usage () { echo " " echo " This script is preparing docker images list based on kubernetes project" echo " Usage:" - echo " ./$(basename $0) <project version> <path to project> [<output list file>]" + echo " ./$(basename $0) <path to project> [<output list file>]" echo " " - echo " Example: ./$(basename $0) onap_3.0.2 /root/oom/kubernetes/onap" + echo " Example: ./$(basename $0) /root/oom/kubernetes/onap" echo " " echo " Dependencies: helm, python-yaml, make" echo " " @@ -47,7 +47,7 @@ import yaml import sys with open("${1}", 'r') as f: - values = yaml.load(f) + values = yaml.load(f, Loader=yaml.SafeLoader) enabled = filter(lambda x: values[x].get('enabled', False) == True, values) print(' '.join(enabled)) @@ -55,26 +55,35 @@ PYP } create_list() { - helm template "${PROJECT_DIR}/../${1}" | grep 'image:\ \|tag_version:\ \|h._image' | + if [ -d "${PROJECT_DIR}/../${1}" ]; then + SUBSYS_DIR="${PROJECT_DIR}/../${1}" + elif [ -d "${PROJECT_DIR}/../common/${1}" ]; then + SUBSYS_DIR="${PROJECT_DIR}/../common/${1}" + else + >&2 echo -e \n" !!! ${1} sybsystem does not exist !!!"\n + fi + helm template "${SUBSYS_DIR}" | grep 'image:\ \|tag_version:\ \|h._image' | sed -e 's/^.*\"h._image\"\ :\ //; s/^.*\"\(.*\)\".*$/\1/' \ -e 's/\x27\|,//g; s/^.*\(image\|tag_version\):\ //' | tr -d '\r' } # Configuration -TAG="${1}" -PROJECT_DIR="${2}" -LIST="${3}" +if [ "${1}" == "-h" ] || [ "${1}" == "--help" ] || [ $# -lt 1 ]; then + usage +fi + +PROJECT_DIR="${1}" +LIST="${2}" LISTS_DIR="$(readlink -f $(dirname ${0}))/../data_lists" HELM_REPO="local http://127.0.0.1:8879" +PROJECT="$(basename ${1})" -if [ "${1}" == "-h" ] || [ "${1}" == "--help" ] || [ $# -lt 2 ]; then - usage -elif [ ! -f "${PROJECT_DIR}/../Makefile" ]; then +if [ ! -f "${PROJECT_DIR}/../Makefile" ]; then echo "Wrong path to project directory entered" exit 1 elif [ -z "${LIST}" ]; then mkdir -p ${LISTS_DIR} - LIST="${LISTS_DIR}/${TAG}-docker_images.list" + LIST="${LISTS_DIR}/${PROJECT}_docker_images.list" fi if [ -e "${LIST}" ]; then @@ -82,8 +91,6 @@ if [ -e "${LIST}" ]; then MSG="$(realpath ${LIST}) already existed\nCreated backup $(realpath ${LIST}).bk\n" fi -PROJECT="$(basename ${2})" - # Setup helm if pgrep -x "helm" > /dev/null; then echo "helm is already running" @@ -106,12 +113,17 @@ popd # Create the list from all enabled subsystems echo "Creating the list..." if [ "${PROJECT}" == "onap" ]; then + COMMENT="OOM commit $(git --git-dir="${PROJECT_DIR}/../../.git" rev-parse HEAD)" for subsystem in `parse_yaml "${PROJECT_DIR}/values.yaml"`; do create_list ${subsystem} - done + done | sort -u > ${LIST} else - create_list ${PROJECT} -fi | sort -u > ${LIST} + COMMENT="${PROJECT}" + create_list ${PROJECT} | sort -u > ${LIST} +fi + +# Add comment reffering to the project +sed -i "1i# generated from ${COMMENT}" "${LIST}" echo -e ${MSG} echo -e 'The list has been created:\n '"${LIST}" diff --git a/build/creating_data/download-bin-tools.sh b/build/creating_data/download-bin-tools.sh deleted file mode 100755 index 327e210f..00000000 --- a/build/creating_data/download-bin-tools.sh +++ /dev/null @@ -1,60 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018-2019 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -usage () { - echo "Usage:" - echo -e "./$(basename $0) [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) ./downloads" -} - -if [ "${1}" == "-h" ] || [ -z "${1}" ] ; then - usage - exit 1 -else - OUTDIR="${1}" -fi - -# we are keeping just dublin support in dublin branch -KUBECTL_VERSION=${KUBECTL_VERSION:-1.13.5} -HELM_VERSION=${HELM_VERSION:-2.12.3} -RKE_VERSION=${RKE_VERSION:-0.2.1} - -mkdir -p "$OUTDIR" -cd "$OUTDIR" - -download() { - url="$1" - url_file="${url%%\?*}" - file=$(basename "$url_file") - echo "Downloading $url" - curl -s --retry 5 -y 10 -Y 10 --location "$url" -o "$file" -} - -download "https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - -download "https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz" -tar -xf ./helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64/helm -O > helm -rm -f ./helm-v${HELM_VERSION}-linux-amd64.tar.gz - -download "https://github.com/rancher/rke/releases/download/v${RKE_VERSION}/rke_linux-amd64" -mv rke_linux-amd64 rke - -chmod a+x ./helm ./kubectl ./rke - -exit 0 diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh deleted file mode 100755 index c0a0bed1..00000000 --- a/build/creating_data/download-docker-images.sh +++ /dev/null @@ -1,39 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - LIST_FILE="docker_image_list.txt" -fi - -echo "Download all images" - -lines=$(clean_list "$LIST_FILE" | wc -l) -line=1 -for image in $(clean_list "$LIST_FILE"); do - echo "== pkg #$line of $lines ==" - echo "$image" - retry docker -l error pull "$image" - line=$((line+1)) -done diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh deleted file mode 100755 index f687fda1..00000000 --- a/build/creating_data/download-files.sh +++ /dev/null @@ -1,50 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -for line in $(clean_list "$LIST_FILE"); do - # www.springframework.org/schema/tool/spring-tool-4.3.xsd - file="${line%%\?*}" - filename=$(basename "$file") - echo "Downloading $cnt / $lines: $file" - # following curl params are ensurring 5 reties and cut-off if connectivity will - # drop below 10b/10s - curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$filename" &>/dev/null - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-git-repos.sh b/build/creating_data/download-git-repos.sh deleted file mode 100755 index 7853a141..00000000 --- a/build/creating_data/download-git-repos.sh +++ /dev/null @@ -1,56 +0,0 @@ -#! /usr/bin/env bash -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -# fail fast -set -e - -usage () { - echo "Usage:" - echo -e "./$(basename $0) <repository list> [destination directory]\n" - echo "Examples:" - echo " ./$(basename $0) onap_3.0.x-git_repos.list ./git-repo" -} - -LIST="${1}" - -if [[ -z "${LIST}" ]]; then - echo "Missing argument for repository list" - exit 1 -fi - -OUTDIR="${2}" -if [[ -z "${OUTDIR}" ]]; then - OUTDIR="./git-repo" -fi - -mkdir -p "${OUTDIR}" -cd "${OUTDIR}" - - -while IFS=" " read -r REPO BRANCH remainder -do - if [[ -z "${BRANCH}" ]]; then - git clone https://${REPO} --bare ${REPO} - else - git clone -b ${BRANCH} --single-branch https://${REPO} --bare ${REPO} - fi -done < <(awk '$1 ~ /^[^;#]/' ${LIST}) - - -exit 0 diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh deleted file mode 100755 index 1144c66e..00000000 --- a/build/creating_data/download-http-files.sh +++ /dev/null @@ -1,51 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -for line in $(clean_list "$LIST_FILE"); do - # www.springframework.org/schema/tool/spring-tool-4.3.xsd - file="${line%%\?*}" - echo "Downloading $cnt / $lines: $file" - fdir=$(dirname "$file") - mkdir -p $outdir/$fdir - # following curl params are ensurring 5 reties and cut-off if connectivity will - # drop below 10b/10s - curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh deleted file mode 100755 index 191dd5df..00000000 --- a/build/creating_data/download-npm-pkgs.sh +++ /dev/null @@ -1,42 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" - -if [[ -z "$LIST_FILE" ]]; then - LIST_FILE="all_npm_list.txt" -fi - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing arg outdir" - exit 1 -fi - -mkdir -p "$outdir" -cd "$outdir" -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 -for line in $(clean_list "$LIST_FILE"); do - echo "== pkg #$cnt of $lines ==" - npm pack $line - cnt=$((cnt+1)) -done
\ No newline at end of file diff --git a/build/creating_data/download-pip.sh b/build/creating_data/download-pip.sh deleted file mode 100755 index 7ab4b0c8..00000000 --- a/build/creating_data/download-pip.sh +++ /dev/null @@ -1,48 +0,0 @@ -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="$1" -if [[ -z "$LIST_FILE" ]]; then - echo "Missing list file" - exit 1 -fi -LIST_FILE=$(readlink -f "$LIST_FILE") - - -outdir="$2" -if [[ -z "$outdir" ]]; then - echo "Missing output directory" - exit 1 -fi - -lines=$(clean_list "$LIST_FILE" | wc -l) -cnt=1 - -# create output dir if not exists -mkdir -p "$outdir" - -cd "$outdir" -for line in $(clean_list "$LIST_FILE"); do - echo "Downloading $cnt / $lines: $line" - pip download $line - cnt=$((cnt+1)) -done diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh deleted file mode 100755 index 0a72d15b..00000000 --- a/build/creating_data/save-docker-images.sh +++ /dev/null @@ -1,59 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# Load common-functions library -. $(dirname ${0})/../common-functions.sh - -LIST_FILE="${1}" -IMG_DIR="${2}" - -if [[ -z "$IMG_DIR" ]]; then - IMG_DIR="./images" -fi - -echo "Creating ${IMG_DIR}" -if [[ ! -d "${IMG_DIR}" ]]; then - mkdir -p "${IMG_DIR}" -fi - -save_image() { - local name_tag=$1 - echo "$name_tag" - local img_name=$(echo "${name_tag}" | tr /: __) - local img_path="${IMG_DIR}/${img_name}.tar" - - if [[ ! -f "${img_path}" ]] ; then - echo "[DEBUG] save ${name_tag} to ${img_path}" - echo "${name_tag}" >> $IMG_DIR/_image_list.txt - retry docker -l error save -o "${img_path}" ${name_tag} - else - echo "[DEBUG] ${name_tag} already saved" - fi -} - -echo "Save all images" -line=1 -lines=$(clean_list "$LIST_FILE" | wc -l) -for image in $(clean_list "$LIST_FILE"); do - echo "== pkg #$line of $lines ==" - save_image "${image}" - line=$((line+1)) -done
\ No newline at end of file diff --git a/build/data_lists/infra_bin_utils.list b/build/data_lists/infra_bin_utils.list new file mode 100644 index 00000000..8de616e9 --- /dev/null +++ b/build/data_lists/infra_bin_utils.list @@ -0,0 +1,3 @@ +https://storage.googleapis.com/kubernetes-release/release/v1.13.5/bin/linux/amd64/kubectl +https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz +https://github.com/rancher/rke/releases/download/v0.2.1/rke_linux-amd64 diff --git a/build/data_lists/onap_docker_images.list b/build/data_lists/onap_docker_images.list index 3ac67ed6..451f6172 100644 --- a/build/data_lists/onap_docker_images.list +++ b/build/data_lists/onap_docker_images.list @@ -1,13 +1,13 @@ -# generated from OOM commit 94664fb4457c61076cc7e65ed40dda5cf696bcbe +# generated from OOM commit 0b904977dde761d189874d6dc6c527cd45928d92 alpine:3.6 busybox crunchydata/crunchy-pgpool:centos7-10.4-2.0.0 crunchydata/crunchy-postgres:centos7-10.3-1.8.2 crunchydata/crunchy-postgres:centos7-10.4-2.0.0 docker.elastic.co/beats/filebeat:5.5.0 +docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.2 docker.elastic.co/elasticsearch/elasticsearch:5.5.0 docker.elastic.co/elasticsearch/elasticsearch:6.6.2 -docker.elastic.co/elasticsearch/elasticsearch-oss:6.6.2 docker.elastic.co/kibana/kibana:5.5.0 docker.elastic.co/kibana/kibana:6.6.2 docker.elastic.co/logstash/logstash:5.4.3 @@ -45,8 +45,8 @@ nexus3.onap.org:10001/onap/aaf/aaf_locate:2.1.13 nexus3.onap.org:10001/onap/aaf/aaf_oauth:2.1.13 nexus3.onap.org:10001/onap/aaf/aaf_service:2.1.13 nexus3.onap.org:10001/onap/aaf/distcenter:4.0.0 -nexus3.onap.org:10001/onap/aaf/sms:4.0.0 nexus3.onap.org:10001/onap/aaf/smsquorumclient:4.0.0 +nexus3.onap.org:10001/onap/aaf/sms:4.0.0 nexus3.onap.org:10001/onap/aaf/testcaservice:4.0.0 nexus3.onap.org:10001/onap/aai/esr-gui:1.4.0 nexus3.onap.org:10001/onap/aai/esr-server:1.4.0 @@ -54,23 +54,22 @@ nexus3.onap.org:10001/onap/aai-graphadmin:1.1.0 nexus3.onap.org:10001/onap/aai-resources:1.4.0 nexus3.onap.org:10001/onap/aai-schema-service:1.0.6 nexus3.onap.org:10001/onap/aai-traversal:1.4.1 -nexus3.onap.org:10001/onap/admportal-sdnc-image:1.5.1 -nexus3.onap.org:10001/onap/appc-cdt-image:1.5.0 -nexus3.onap.org:10001/onap/appc-image:1.5.0 +nexus3.onap.org:10001/onap/admportal-sdnc-image:1.5.2 +nexus3.onap.org:10001/onap/appc-cdt-image:1.5.1 +nexus3.onap.org:10001/onap/appc-image:1.5.1 nexus3.onap.org:10001/onap/babel:1.4.2 nexus3.onap.org:10001/onap/ccsdk-ansible-server-image:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-apps-ms-neng:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-blueprintsprocessor:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-cds-ui-server:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-commandexecutor:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-controllerblueprints:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-dgbuilder-image:0.4.2 -nexus3.onap.org:10001/onap/ccsdk-sdclistener:0.4.2 -nexus3.onap.org:10001/onap/champ:1.4.0 -nexus3.onap.org:10001/onap/clamp:4.0.2 +nexus3.onap.org:10001/onap/ccsdk-apps-ms-neng:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-blueprintsprocessor:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-cds-ui-server:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-commandexecutor:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-controllerblueprints:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-dgbuilder-image:0.4.3 +nexus3.onap.org:10001/onap/ccsdk-sdclistener:0.4.3 nexus3.onap.org:10001/onap/clamp-dashboard-kibana:4.0.1 nexus3.onap.org:10001/onap/clamp-dashboard-logstash:4.0.1 -nexus3.onap.org:10001/onap/cli:2.0.4 +nexus3.onap.org:10001/onap/clamp:4.0.2 +nexus3.onap.org:10001/onap/cli:3.0.0 nexus3.onap.org:10001/onap/data-router:1.3.3 nexus3.onap.org:10001/onap/data-router:1.4.0 nexus3.onap.org:10001/onap/dcae-be:1.3.0 @@ -90,24 +89,25 @@ nexus3.onap.org:10001/onap/externalapi/nbi:4.0.0 nexus3.onap.org:10001/onap/gizmo:1.4.0 nexus3.onap.org:10001/onap/holmes/engine-management:1.2.5 nexus3.onap.org:10001/onap/holmes/rule-management:1.2.6 +nexus3.onap.org:10001/onap/champ:1.4.0 nexus3.onap.org:10001/onap/modeling/genericparser:1.0.2 nexus3.onap.org:10001/onap/model-loader:1.4.0 nexus3.onap.org:10001/onap/msb/msb_apigateway:1.2.4 nexus3.onap.org:10001/onap/msb/msb_discovery:1.2.3 -nexus3.onap.org:10001/onap/multicloud/azure:1.2.1 -nexus3.onap.org:10001/onap/multicloud/framework:1.3.1 +nexus3.onap.org:10001/onap/multicloud/azure:1.2.2 nexus3.onap.org:10001/onap/multicloud/framework-artifactbroker:1.3.3 -nexus3.onap.org:10001/onap/multicloud/k8s:0.2.0 -nexus3.onap.org:10001/onap/multicloud/openstack-fcaps:1.3.2 -nexus3.onap.org:10001/onap/multicloud/openstack-lenovo:1.3.1 -nexus3.onap.org:10001/onap/multicloud/openstack-ocata:1.3.2 -nexus3.onap.org:10001/onap/multicloud/openstack-pike:1.3.2 -nexus3.onap.org:10001/onap/multicloud/openstack-starlingx:1.3.2 -nexus3.onap.org:10001/onap/multicloud/openstack-windriver:1.3.2 +nexus3.onap.org:10001/onap/multicloud/framework:1.3.3 +nexus3.onap.org:10001/onap/multicloud/k8s:0.4.0 +nexus3.onap.org:10001/onap/multicloud/openstack-fcaps:1.3.4 +nexus3.onap.org:10001/onap/multicloud/openstack-lenovo:1.3.4 +nexus3.onap.org:10001/onap/multicloud/openstack-ocata:1.3.4 +nexus3.onap.org:10001/onap/multicloud/openstack-pike:1.3.4 +nexus3.onap.org:10001/onap/multicloud/openstack-starlingx:1.3.4 +nexus3.onap.org:10001/onap/multicloud/openstack-windriver:1.3.4 nexus3.onap.org:10001/onap/multicloud/vio:1.3.1 -nexus3.onap.org:10001/onap/music/cassandra_3_11:3.0.24 nexus3.onap.org:10001/onap/music/cassandra_job:3.0.24 nexus3.onap.org:10001/onap/music/cassandra_music:3.0.0 +nexus3.onap.org:10001/onap/music/cassandra_3_11:3.0.24 nexus3.onap.org:10001/onap/music/music:3.0.24 nexus3.onap.org:10001/onap/network-discovery:1.5.1 nexus3.onap.org:10001/onap/oom/kube2msb:1.1.0 @@ -118,17 +118,17 @@ nexus3.onap.org:10001/onap/optf-cmso-ticketmgt:2.0.0 nexus3.onap.org:10001/onap/optf-cmso-topology:2.0.0 nexus3.onap.org:10001/onap/optf-has:1.3.0 nexus3.onap.org:10001/onap/optf-osdf:1.3.0 -nexus3.onap.org:10001/onap/org.onap.ccsdk.dashboard.ccsdk-app-os:1.1.0-SNAPSHOT-latest +nexus3.onap.org:10001/onap/org.onap.ccsdk.dashboard.ccsdk-app-os:1.1.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main:1.1.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.snmptrap:1.4.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.collectors.ves.vescollector:1.4.4 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.cm-container:1.6.2 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.consul-loader-container:1.0.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.healthcheck-container:1.2.4 -nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.k8s-bootstrap-container:1.4.17 +nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.k8s-bootstrap-container:1.4.18 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.multisite-init-container:1.0.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.redis-cluster-container:1.0.0 -nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.tca-cdap-container:1.1.1 +nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.tca-cdap-container:1.1.2 nexus3.onap.org:10001/onap/org.onap.dcaegen2.deployments.tls-init-container:1.0.3 nexus3.onap.org:10001/onap/org.onap.dcaegen2.platform.configbinding.app-app:2.3.0 nexus3.onap.org:10001/onap/org.onap.dcaegen2.platform.deployment-handler:4.0.1 @@ -152,19 +152,19 @@ nexus3.onap.org:10001/onap/portal-app:2.5.0 nexus3.onap.org:10001/onap/portal-db:2.5.0 nexus3.onap.org:10001/onap/portal-sdk:2.5.0 nexus3.onap.org:10001/onap/portal-wms:2.5.0 -nexus3.onap.org:10001/onap/sdc-backend:1.4.0 -nexus3.onap.org:10001/onap/sdc-backend-init:1.4.0 -nexus3.onap.org:10001/onap/sdc-cassandra-init:1.4.0 -nexus3.onap.org:10001/onap/sdc-elasticsearch:1.4.0 -nexus3.onap.org:10001/onap/sdc-frontend:1.4.0 -nexus3.onap.org:10001/onap/sdc-init-elasticsearch:1.4.0 -nexus3.onap.org:10001/onap/sdc-kibana:1.4.0 -nexus3.onap.org:10001/onap/sdc-onboard-backend:1.4.0 -nexus3.onap.org:10001/onap/sdc-onboard-cassandra-init:1.4.0 -nexus3.onap.org:10001/onap/sdnc-ansible-server-image:1.5.1 -nexus3.onap.org:10001/onap/sdnc-dmaap-listener-image:1.5.1 -nexus3.onap.org:10001/onap/sdnc-image:1.5.1 -nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:1.5.1 +nexus3.onap.org:10001/onap/sdc-backend-init:1.4.1 +nexus3.onap.org:10001/onap/sdc-backend:1.4.1 +nexus3.onap.org:10001/onap/sdc-cassandra-init:1.4.1 +nexus3.onap.org:10001/onap/sdc-elasticsearch:1.4.1 +nexus3.onap.org:10001/onap/sdc-frontend:1.4.1 +nexus3.onap.org:10001/onap/sdc-init-elasticsearch:1.4.1 +nexus3.onap.org:10001/onap/sdc-kibana:1.4.1 +nexus3.onap.org:10001/onap/sdc-onboard-backend:1.4.1 +nexus3.onap.org:10001/onap/sdc-onboard-cassandra-init:1.4.1 +nexus3.onap.org:10001/onap/sdnc-ansible-server-image:1.5.2 +nexus3.onap.org:10001/onap/sdnc-dmaap-listener-image:1.5.2 +nexus3.onap.org:10001/onap/sdnc-image:1.5.2 +nexus3.onap.org:10001/onap/sdnc-ueb-listener-image:1.5.2 nexus3.onap.org:10001/onap/search-data-service:1.3.1 nexus3.onap.org:10001/onap/search-data-service:1.4.3 nexus3.onap.org:10001/onap/service-decomposition:1.5.1 @@ -182,8 +182,8 @@ nexus3.onap.org:10001/onap/so/vnfm-adapter:1.4.1 nexus3.onap.org:10001/onap/sparky-be:1.4.0 nexus3.onap.org:10001/onap/spike:1.4.0 nexus3.onap.org:10001/onap/testsuite:1.4.0 -nexus3.onap.org:10001/onap/usecase-ui:1.2.2 nexus3.onap.org:10001/onap/usecase-ui-server:1.2.1 +nexus3.onap.org:10001/onap/usecase-ui:1.2.2 nexus3.onap.org:10001/onap/validation:1.3.1 nexus3.onap.org:10001/onap/vfc/catalog:1.3.1 nexus3.onap.org:10001/onap/vfc/db:1.3.0 @@ -202,11 +202,11 @@ nexus3.onap.org:10001/onap/vfc/wfengine-activiti:1.3.0 nexus3.onap.org:10001/onap/vfc/wfengine-mgrservice:1.3.0 nexus3.onap.org:10001/onap/vfc/ztesdncdriver:1.3.0 nexus3.onap.org:10001/onap/vfc/ztevnfmdriver:1.3.1 -nexus3.onap.org:10001/onap/vid:4.0.0 -nexus3.onap.org:10001/onap/vnfsdk/refrepo:1.2.0 -nexus3.onap.org:10001/onap/workflow-backend:1.4.0 -nexus3.onap.org:10001/onap/workflow-frontend:1.4.0 -nexus3.onap.org:10001/onap/workflow-init:1.4.0 +nexus3.onap.org:10001/onap/vid:4.2.0 +nexus3.onap.org:10001/onap/vnfsdk/refrepo:1.3.0 +nexus3.onap.org:10001/onap/workflow-backend:1.4.1 +nexus3.onap.org:10001/onap/workflow-frontend:1.4.1 +nexus3.onap.org:10001/onap/workflow-init:1.4.1 nexus3.onap.org:10001/sonatype/nexus:2.14.8-01 nexus3.onap.org:10001/zookeeper:3.4 oomk8s/mariadb-client-init:3.0.0 diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list index ba5cdf7e..48029bc2 100644 --- a/build/data_lists/onap_pip_packages.list +++ b/build/data_lists/onap_pip_packages.list @@ -5,8 +5,7 @@ Flask==1.0.3 idna==2.8 itsdangerous==1.1.0 Jinja2==2.10.1 -MarkupSafe==1.1.1 +MarkupSafe==0.23 requests==2.22.0 -setuptools==40.7.1 urllib3==1.25.3 Werkzeug==0.15.4 diff --git a/build/data_lists/onap_rpm.list b/build/data_lists/onap_rpm.list new file mode 100644 index 00000000..4595d4b1 --- /dev/null +++ b/build/data_lists/onap_rpm.list @@ -0,0 +1,21 @@ +containerd.io-1.2.5-3.1.el7.x86_64 +container-selinux-1.12.5-14.el7.x86_64 +container-selinux-2.95-2.el7_6.noarch +docker-ce-18.09.5-3.el7.x86_64 +docker-ce-cli-18.09.6-3.el7.x86_64 +gssproxy-0.7.0-21.el7.x86_64 +keyutils-1.5.8-3.el7.x86_64 +libbasicobjects-0.1.1-32.el7.x86_64 +libcollection-0.7.0-32.el7.x86_64 +libevent-2.0.21-4.el7.x86_64 +libini_config-1.3.1-32.el7.x86_64 +libnfsidmap-0.25-19.el7.x86_64 +libpath_utils-0.2.1-32.el7.x86_64 +libref_array-0.1.5-32.el7.x86_64 +libverto-libevent-0.2.5-4.el7.x86_64 +nfs-utils-1.3.0-0.61.el7.x86_64 +python-docker-py-1.10.6-9.el7_6.noarch +python-docker-pycreds-0.3.0-9.el7_6.noarch +python-ipaddress-1.0.16-2.el7.noarch +python-jsonpointer-1.9-2.el7.noarch +python-websocket-client-0.32.0-116.el7.noarch diff --git a/build/download/base.py b/build/download/base.py index 5bcd0ef6..d8b44839 100644 --- a/build/download/base.py +++ b/build/download/base.py @@ -38,7 +38,8 @@ def load_list(item_list): :return: set of items from file """ with open(item_list, 'r') as f: - return {item for item in (line.strip() for line in f) if item} + return {item for item in (line.strip() for line in f) + if item and not item.startswith('#')} def init_progress(items_name): diff --git a/build/download/docker_images.py b/build/download/docker_images.py index e4e742b3..d8138dd6 100755 --- a/build/download/docker_images.py +++ b/build/download/docker_images.py @@ -180,7 +180,7 @@ def download_docker_image(image, save, output_dir, docker_client): if save: save_image(image, pulled_image, output_dir) except Exception as err: - log.error('Error downloading {}: {}'.format(image, err)) + log.exception('Error downloading {}: {}'.format(image, err)) raise err @@ -195,10 +195,10 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): :return: None """ try: - docker_client = docker.client.DockerClient(version='auto') + # big timeout in case of massive images like pnda-mirror-container:5.0.0 (11.4GB) + docker_client = docker.client.DockerClient(version='auto', timeout=300) except docker.errors.DockerException as err: - log.error(err) - log.error('Error creating docker client. Check if is docker installed and running' + log.exception('Error creating docker client. Check if is docker installed and running' ' or if you have right permissions.') raise err @@ -221,14 +221,12 @@ def download(image_list, save, output_dir, check_mode, progress, workers=3): missing_images['not_saved'] - missing_images['not_pulled'], None, output_dir, docker_client) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} images were not downloaded'.format(error_count)) missing_images = missing(docker_client, target_images, save, output_dir) log.info(check_table(merge_dict_sets(missing_images), missing_images, save)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): @@ -256,11 +254,13 @@ def run_cli(): progress = base.init_progress('Docker images') if not args.check else None try: - sys.exit(download(args.image_list, args.save, args.output_dir, args.check, - progress, args.workers)) + download(args.image_list, args.save, args.output_dir, args.check, + progress, args.workers) except docker.errors.DockerException: - log.error('Irrecoverable error detected.') + log.exception('Irrecoverable error detected.') sys.exit(1) + except RuntimeError as err: + log.exception(err) if __name__ == '__main__': diff --git a/build/download/download.py b/build/download/download.py new file mode 100755 index 00000000..0d8912ee --- /dev/null +++ b/build/download/download.py @@ -0,0 +1,173 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import datetime +import timeit + +import base +import docker_images +import git_repos +import http_files +import npm_packages +import pypi_packages +import rpm_packages + +log = logging.getLogger(name=__name__) + +def parse_args(): + parser=argparse.ArgumentParser(description='Download data from lists') + list_group = parser.add_argument_group() + list_group.add_argument('--docker', action='append', nargs='+', default=[], + metavar=('list', 'dir-name'), + help='Docker type list. If second argument is specified ' + 'it is treated as directory where images will be saved ' + 'otherwise only pull operation is executed') + list_group.add_argument('--http', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='Http type list and directory to save downloaded files') + list_group.add_argument('--npm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='npm type list and directory to save downloaded files') + list_group.add_argument('--rpm', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='rpm type list and directory to save downloaded files') + list_group.add_argument('--git', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='git repo type list and directory to save downloaded files') + list_group.add_argument('--pypi', action='append', nargs=2, default=[], + metavar=('list', 'dir-name'), + help='pypi packages type list and directory to save downloaded files') + parser.add_argument('--npm-registry', default='https://registry.npmjs.org', + help='npm registry to use (default: https://registry.npmjs.org)') + parser.add_argument('--check', '-c', action='store_true', default=False, + help='Check what is missing. No download.') + parser.add_argument('--debug', action='store_true', default=False, + help='Turn on debug output') + + args = parser.parse_args() + + for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'): + if getattr(args, arg): + return args + + parser.error('One of --docker, --npm, --http, --rpm, --git must be specified') + + +def run_cli(): + args = parse_args() + + console_handler = logging.StreamHandler(sys.stdout) + console_formatter = logging.Formatter('%(message)s') + console_handler.setFormatter(console_formatter) + now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + log_file = 'download_data-{}.log'.format(now) + file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s" + + if args.debug: + logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format) + else: + logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format) + root_logger = logging.getLogger() + root_logger.addHandler(console_handler) + + list_with_errors = [] + timer_start = timeit.default_timer() + + for docker_list in args.docker: + log.info('Processing {}.'.format(docker_list[0])) + progress = None if args.check else base.init_progress('docker images') + save = False + if len(docker_list) > 1: + save = True + else: + docker_list.append(None) + try: + docker_images.download(docker_list[0], save, + docker_list[1], args.check, progress) + except RuntimeError: + list_with_errors.append(docker_list[0]) + + for http_list in args.http: + progress = None if args.check else base.init_progress('http files') + log.info('Processing {}.'.format(http_list[0])) + try: + http_files.download(http_list[0], http_list[1], args.check, + progress) + except RuntimeError: + list_with_errors.append(http_list[0]) + + for npm_list in args.npm: + progress = None if args.check else base.init_progress('npm packages') + log.info('Processing {}.'.format(npm_list[0])) + try: + npm_packages.download(npm_list[0], args.npm_registry, npm_list[1], + args.check, progress) + except RuntimeError: + list_with_errors.append(npm_list[0]) + + for rpm_list in args.rpm: + if args.check: + log.info('Check mode for rpm packages is not implemented') + break + log.info('Processing {}.'.format(rpm_list[0])) + try: + rpm_packages.download(rpm_list[0], rpm_list[1]) + except RuntimeError: + list_with_errors.append(rpm_list[0]) + + for git_list in args.git: + if args.check: + log.info('Check mode for git repositories is not implemented') + break + progress = None if args.check else base.init_progress('git repositories') + log.info('Processing {}.'.format(git_list[0])) + try: + git_repos.download(git_list[0], git_list[1], progress) + except RuntimeError: + list_with_errors.append(git_list[0]) + + for pypi_list in args.pypi: + if args.check: + log.info('Check mode for pypi packages is not implemented') + break + progress = None if args.check else base.init_progress('pypi packages') + log.info('Processing {}.'.format(pypi_list[0])) + try: + pypi_packages.download(pypi_list[0], pypi_list[1], progress) + except RuntimeError: + list_with_errors.append(pypi_list[0]) + + e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start) + log.info(timeit.default_timer() - timer_start) + log.info('Execution ended. Total elapsed time {}'.format(e_time)) + + if list_with_errors: + log.error('Errors encountered while processing these lists:' + '\n{}'.format('\n'.join(list_with_errors))) + sys.exit(1) + + + +if __name__ == '__main__': + run_cli() diff --git a/build/download/git_repos.py b/build/download/git_repos.py index e388e94c..1d8c2979 100755 --- a/build/download/git_repos.py +++ b/build/download/git_repos.py @@ -21,6 +21,7 @@ import argparse import subprocess +import shutil import logging import sys import os @@ -45,10 +46,9 @@ def download(git_list, dst_dir, progress): if not base.check_tool('git'): log.error('ERROR: git is not installed') progress.finish(dirty=True) - return 1 + raise RuntimeError('git missing') - git_set = {tuple(item.split()) for item in base.load_list(git_list) - if not item.startswith('#')} + git_set = {tuple(item.split()) for item in base.load_list(git_list)} error_count = 0 @@ -64,19 +64,20 @@ def download(git_list, dst_dir, progress): clone_repo(dst, *repo) progress.update(progress.value + 1) except subprocess.CalledProcessError as err: - log.error(err.output.decode()) + if os.path.isdir(dst): + shutil.rmtree(dst) + log.exception(err.output.decode()) error_count += 1 base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} were not downloaded. Check logs for details'.format(error_count)) - return error_count - + raise RuntimeError('Download unsuccessful') def run_cli(): parser = argparse.ArgumentParser(description='Download git repositories from list') parser.add_argument('git_list', metavar='git-list', - help='File with list of npm packages to download.') + help='File with list of git repos to download.') parser.add_argument('--output-dir', '-o', default=os.getcwd(), help='Download destination') @@ -85,8 +86,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') progress = base.init_progress('git repositories') - - sys.exit(download(args.git_list, args.output_dir, progress)) + try: + download(args.git_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/http_files.py b/build/download/http_files.py index f5b1e59a..c83158d6 100755 --- a/build/download/http_files.py +++ b/build/download/http_files.py @@ -83,7 +83,7 @@ def download(data_list, dst_dir, check, progress, workers=None): if check: log.info(base.simple_check_table(file_set, missing_files)) - return 0 + return skipping = file_set - missing_files @@ -91,12 +91,11 @@ def download(data_list, dst_dir, check, progress, workers=None): error_count = base.run_concurrent(workers, progress, download_file, missing_files, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} files were not downloaded. Check log for specific failures.'.format(error_count)) + raise RuntimeError() - base.finish_progress(progress, error_count, log) - - return error_count def run_cli(): """ @@ -123,7 +122,10 @@ def run_cli(): progress = base.init_progress('http files') if not args.check else None - sys.exit(download(args.file_list, args.output_dir, args.check, progress, args.workers)) + try: + download(args.file_list, args.output_dir, args.check, progress, args.workers) + except RuntimeError: + sys.exit(1) if __name__ == '__main__': diff --git a/build/download/npm_packages.py b/build/download/npm_packages.py index c174e2c1..70c03ad8 100755 --- a/build/download/npm_packages.py +++ b/build/download/npm_packages.py @@ -57,7 +57,7 @@ def download_npm(npm, registry, dst_dir): except Exception as err: if os.path.isfile(dst_path): os.remove(dst_path) - log.error('Failed: {}: {}'.format(npm, err)) + log.exception('Failed: {}'.format(npm)) raise err log.info('Downloaded: {}'.format(npm)) @@ -81,12 +81,10 @@ def download(npm_list, registry, dst_dir, check_mode, progress=None, workers=Non base.start_progress(progress, len(npm_set), skipping, log) error_count = base.run_concurrent(workers, progress, download_npm, missing_npms, registry, dst_dir) + base.finish_progress(progress, error_count, log) if error_count > 0: log.error('{} packages were not downloaded. Check log for specific failures.'.format(error_count)) - - base.finish_progress(progress, error_count, log) - - return error_count + raise RuntimeError() def run_cli(): diff --git a/build/download/pypi_packages.py b/build/download/pypi_packages.py new file mode 100755 index 00000000..951003c5 --- /dev/null +++ b/build/download/pypi_packages.py @@ -0,0 +1,88 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +# COPYRIGHT NOTICE STARTS HERE + +# Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# COPYRIGHT NOTICE ENDS HERE + +import argparse +import logging +import sys +import subprocess +import os +from retrying import retry + +import base + +log = logging.getLogger(name=__name__) + +@retry(stop_max_attempt_number=5, wait_fixed=5000) +def download_package(package_name, dst_dir): + command = 'pip download --dest {} {}'.format(dst_dir, package_name) + log.info('Running: {}'.format(command)) + log.info(subprocess.check_output(command.split(), stderr=subprocess.STDOUT).decode()) + log.info('Downloaded: {}'.format(package_name)) + + +def download(pypi_list, dst_dir, progress): + if not base.check_tool('pip'): + log.error('ERROR: pip is not installed') + progress.finish(dirty=True) + raise RuntimeError('pip missing') + + pypi_set = base.load_list(pypi_list) + + error_count = 0 + + base.start_progress(progress, len(pypi_set), [], log) + + for package in pypi_set: + try: + download_package(package, dst_dir) + except subprocess.CalledProcessError as err: + log.exception(err.output.decode()) + error_count += 1 + + progress.update(progress.value + 1) + + base.finish_progress(progress, error_count, log) + if error_count > 0: + log.error('{} packages were not downloaded. Check logs for details'.format(error_count)) + raise RuntimeError('Download unsuccesfull') + + +def run_cli(): + parser = argparse.ArgumentParser(description='Download git repositories from list') + parser.add_argument('pypi_list', metavar='pypi-list', + help='File with list of pypi packages to download.') + parser.add_argument('--output-dir', '-o', default=os.getcwd(), + help='Download destination') + + args = parser.parse_args() + + logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') + + progress = base.init_progress('pypi packages') + try: + download(args.pypi_list, args.output_dir, progress) + except RuntimeError as err: + log.exception(err) + sys.exit(1) + + +if __name__ == '__main__': + run_cli() diff --git a/build/download/rpm_packages.py b/build/download/rpm_packages.py index 7f9700a3..732af0ea 100755 --- a/build/download/rpm_packages.py +++ b/build/download/rpm_packages.py @@ -33,7 +33,7 @@ log = logging.getLogger(name=__name__) def download(rpm_list, dst_dir): if not base.check_tool('yumdownloader'): log.error('ERROR: yumdownloader is not installed') - return 1 + raise RuntimeError('yumdownloader missing') rpm_set = base.load_list(rpm_list) @@ -41,11 +41,10 @@ def download(rpm_list, dst_dir): log.info('Running command: {}'.format(command)) try: subprocess.check_call(command.split()) - log.info('Downloaded') except subprocess.CalledProcessError as err: - log.error(err.output) - return err.returncode - + log.exception(err.output) + raise err + log.info('Downloaded') def run_cli(): @@ -59,7 +58,11 @@ def run_cli(): logging.basicConfig(stream=sys.stdout, level=logging.INFO, format='%(message)s') - sys.exit(download(args.rpm_list, args.output_dir)) + try: + download(args.rpm_list, args.output_dir) + except (subprocess.CalledProcessError, RuntimeError): + sys.exit(1) + if __name__ == '__main__': diff --git a/build/download_offline_data_by_lists.sh b/build/download_offline_data_by_lists.sh deleted file mode 100755 index b2afd172..00000000 --- a/build/download_offline_data_by_lists.sh +++ /dev/null @@ -1,96 +0,0 @@ -#! /usr/bin/env bash - -# COPYRIGHT NOTICE STARTS HERE -# -# Copyright 2018 © Samsung Electronics Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# COPYRIGHT NOTICE ENDS HERE - - -# fail fast -set -e - -usage () { - echo "Usage:" - echo -e "./$(basename $0) <project version>\n" - echo "onap_3.0.0 for casablanca (sign-off 30/11/2018)" - echo "onap_3.0.1 for casablanca maintenance release (sign-off 10/12/2018)" - echo "onap_3.0.2 for latest casablanca with fixed certificates (sign-off 25/04/2019)" - echo "" - echo "Example:" - echo " ./$(basename $0) onap_3.0.2" -} - -# boilerplate -RELATIVE_PATH=./ # relative path from this script to 'common-functions.sh' -if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then - SCRIPT_DIR=$(dirname "${0}") - LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") - . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh -fi - -if [ "${1}" == "-h" ] || [ -z "${1}" ]; then - usage - exit 0 -else - TAG="${1}" -fi - -CTOOLS="${LOCAL_PATH}/creating_data" -LISTS_DIR="${LOCAL_PATH}/data_lists" -DATA_DIR="${LOCAL_PATH}/../../resources" -TOTAL=12 -CURR=1 - -message info "Downloading started: $(date)" - -echo "[Step $((CURR++))/$TOTAL Download collected docker images]" -$CTOOLS/download-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list" - -echo "[Step $((CURR++))/$TOTAL Download docker images for infra-server]" -$CTOOLS/download-docker-images.sh "${LISTS_DIR}/infra_docker_images.list" - -echo "[Step $((CURR++))/$TOTAL Build own nginx image]" -$CTOOLS/create_nginx_image/01create-image.sh "${DATA_DIR}/offline_data/docker_images_infra" - -echo "[Step $((CURR++))/$TOTAL Save docker images from docker cache to tarfiles]" -$CTOOLS/save-docker-images.sh "${LISTS_DIR}/${TAG}-docker_images.list" "${DATA_DIR}/offline_data/docker_images_for_nexus" - -echo "[Step $((CURR++))/$TOTAL Prepare infra related images to infra folder]" -$CTOOLS/save-docker-images.sh "${LISTS_DIR}/infra_docker_images.list" "${DATA_DIR}/offline_data/docker_images_infra" - -echo "[Step $((CURR++))/$TOTAL Download git repos]" -$CTOOLS/download-git-repos.sh "${LISTS_DIR}/onap_3.0.x-git_repos.list" "${DATA_DIR}/git-repo" - -echo "[Step $((CURR++))/$TOTAL Download http files]" -$CTOOLS/download-http-files.sh "${LISTS_DIR}/onap_3.0.x-http_files.list" "${DATA_DIR}/http" - -echo "[Step $((CURR++))/$TOTAL Download npm pkgs]" -$CTOOLS/download-npm-pkgs.sh "${LISTS_DIR}/onap_3.0.x-npm.list" "${DATA_DIR}/offline_data/npm_tar" - -echo "[Step $((CURR++))/$TOTAL Download bin tools]" -$CTOOLS/download-bin-tools.sh "${DATA_DIR}/downloads" - -echo "[Step $((CURR++))/$TOTAL Create RHEL repository]" -$CTOOLS/create-rhel-repo.sh "${DATA_DIR}/pkg/rhel" - -echo "[Step $((CURR++))/$TOTAL Download sdnc-ansible-server packages]" -$CTOOLS/download-pip.sh "${LISTS_DIR}/onap_3.0.x-pip_packages.list" "${DATA_DIR}/offline_data/pypi" -$CTOOLS/download-files.sh "${LISTS_DIR}/deb_packages.list" "${DATA_DIR}/pkg/ubuntu/xenial" - -echo "[Step $((CURR++))/$TOTAL Create APT repository]" -$CTOOLS/create-ubuntu-repo.sh "${DATA_DIR}/pkg/ubuntu/xenial" - -message info "Downloading finished: $(date)" diff --git a/build/fetch_and_patch_charts.sh b/build/fetch_and_patch_charts.sh index 79d7a01e..22d45e66 100755 --- a/build/fetch_and_patch_charts.sh +++ b/build/fetch_and_patch_charts.sh @@ -54,7 +54,7 @@ TOTAL=5 PATCH_FILE=$(realpath "${3}") echo -e "${_G}[Step $((CURR++))/${TOTAL} cloning repo with charts to be patched]${C_}" -git clone "${1}" "${4}" +git clone --recurse-submodules "${1}" "${4}" echo -e "${_G}[Step $((CURR++))/${TOTAL} setting working dir to ${4}]${C_}" pushd "${4}" |