diff options
| -rwxr-xr-x | build/build_nexus_blob.sh | 334 | ||||
| -rwxr-xr-x | build/common-functions.sh | 6 | ||||
| -rwxr-xr-x | build/creating_data/download-docker-images.sh | 34 | ||||
| -rwxr-xr-x | build/creating_data/download-files.sh | 13 | ||||
| -rwxr-xr-x | build/creating_data/download-http-files.sh | 13 | ||||
| -rwxr-xr-x | build/creating_data/download-npm-pkgs.sh | 13 | ||||
| -rwxr-xr-x | build/creating_data/download-pip.sh | 11 | ||||
| -rwxr-xr-x | build/creating_data/save-docker-images.sh | 19 | ||||
| -rw-r--r-- | docs/BuildGuide.rst | 84 |
9 files changed, 231 insertions, 296 deletions
diff --git a/build/build_nexus_blob.sh b/build/build_nexus_blob.sh index dc4b1dd3..4b1697e3 100755 --- a/build/build_nexus_blob.sh +++ b/build/build_nexus_blob.sh @@ -2,7 +2,7 @@ # COPYRIGHT NOTICE STARTS HERE # -# Copyright 2018 © Samsung Electronics Co., Ltd. +# Copyright 2018-2019 © Samsung Electronics Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,37 +18,101 @@ # # COPYRIGHT NOTICE ENDS HERE - ### This script prepares Nexus repositories data blobs for ONAP -# Mandatory variables need to be set in configuration file: -# NXS_SRC_DOCKER_IMG_DIR - resource directory of docker images -# NXS_SRC_NPM_DIR - resource directory of npm packages -# NXS_SRC_PYPI_DIR - resource directory of pypi packages -# NXS_DOCKER_IMG_LIST - list of docker images to be pushed to Nexus repository -# NXS_DOCKER_WO_LIST - list of docker images which uses default repository -# NXS_NPM_LIST - list of npm packages to be published to Nexus repository -# NXS_PYPI_LIST - list of pypi packages to be uploaded to Nexus repository -# NEXUS_DATA_TAR - target tarball of Nexus data path/name -# NEXUS_DATA_DIR - directory used for the Nexus blob build -# NEXUS_IMAGE - Sonatype/Nexus3 docker image which will be used for data blob creation +## The script requires following dependencies are installed: nodejs, jq, docker +## All required resources are expected in the upper directory +## created during download procedure as DATA_DIR +## All lists used must be in project data_lists directory # Fail fast settings set -e +TIMESTAMP="date +'%Y-%m-%d_%H-%M-%S'" +SCRIPT_LOG="/tmp/$(basename $0)_$(eval ${TIMESTAMP}).log" + +# Log everything +exec &> >(tee -a "${SCRIPT_LOG}") + +usage () { + echo " This script is preparing Nexus data blob from docker images and npm and pypi packages" + echo " Usage:" + echo " ./$(basename $0) <project version> [<target>]" + echo " " + echo " Example: ./$(basename $0) onap_3.0.1 /root/nexus_data" + echo " " + echo " Dependencies: nodejs, jq, docker" + echo " " + exit 1 +} + # Nexus repository location NEXUS_DOMAIN="nexus" -NPM_REGISTRY="http://${NEXUS_DOMAIN}:8081/repository/npm-private/" -PYPI_REGISTRY="http://${NEXUS_DOMAIN}:8081/repository/pypi-private/" -DOCKER_REGISTRY="${NEXUS_DOMAIN}:8082" +NEXUS_PORT="8081" +NEXUS_DOCKER_PORT="8082" +NPM_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/npm-private/" +PYPI_REGISTRY="http://${NEXUS_DOMAIN}:${NEXUS_PORT}/repository/pypi-private/" +DOCKER_REGISTRY="${NEXUS_DOMAIN}:${NEXUS_DOCKER_PORT}" +DEFAULT_REGISTRY="docker.io" # Nexus repository credentials NEXUS_USERNAME=admin NEXUS_PASSWORD=admin123 NEXUS_EMAIL=admin@example.org -# Setup simulated domain names to be able to push all in private Nexus repository -SIMUL_HOSTS="docker.elastic.co gcr.io hub.docker.com nexus3.onap.org nexus.onap.org registry.hub.docker.com ${NEXUS_DOMAIN}" +if [ "${1}" == "-h" ] || [ "${1}" == "--help" ] || [ $# -eq 0 ]; then + usage +else + TAG="${1}" +fi + +# Setting paths +LOCAL_PATH="$(readlink -f $(dirname ${0}))" +DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)" + +if [ -z "${2}" ]; then + NEXUS_DATA_DIR="${DATA_DIR}/nexus_data" +else + NEXUS_DATA_DIR="${2}" +fi + +# Setup directory with resources lists +LISTS_DIR="${LOCAL_PATH}/data_lists" + +# Setup directories with resources for docker, npm and pypi +NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus" +NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar" +NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi" + +# Setup specific resources list based on the tag provided +NXS_DOCKER_IMG_LIST="${LISTS_DIR}/${TAG}-docker_images.list" +NXS_NPM_LIST="${LISTS_DIR}/$(sed 's/.$/x/' <<< ${TAG})-npm.list" +NXS_PYPI_LIST="${LISTS_DIR}/$(sed 's/.$/x/' <<< ${TAG})-pip_packages.list" + +# Setup Nexus image used for build and install infra +INFRA_LIST="${LISTS_DIR}/infra_docker_images.list" +NEXUS_IMAGE="$(grep sonatype/nexus3 ${INFRA_LIST})" +NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})" + +# Setup default ports published to host as docker registry +PUBLISHED_PORTS="-p ${NEXUS_PORT}:${NEXUS_PORT} -p ${NEXUS_DOCKER_PORT}:${NEXUS_DOCKER_PORT}" + +# Setup additional ports published to host based on simulated docker registries +for REGISTRY in $(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true); do + if [[ ${REGISTRY} != *":"* ]]; then + if [[ ${PUBLISHED_PORTS} != *"80:${NEXUS_DOCKER_PORT}"* ]]; then + PUBLISHED_PORTS="${PUBLISHED_PORTS} -p 80:${NEXUS_DOCKER_PORT}" + fi + else + REGISTRY_PORT="$(sed 's/^.*\:\([[:digit:]]*\)$/\1/' <<< ${REGISTRY})" + if [[ ${PUBLISHED_PORTS} != *"${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}"* ]]; then + PUBLISHED_PORTS="${PUBLISHED_PORTS} -p ${REGISTRY_PORT}:${NEXUS_DOCKER_PORT}" + fi + fi +done + +# Setup simulated domain names to be able to push all to private Nexus repository +SIMUL_HOSTS="$(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$// ; s/:.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY} || true) ${NEXUS_DOMAIN}" # Nexus repository configuration setup NEXUS_CONFIG_GROOVY='import org.sonatype.nexus.security.realm.RealmManager @@ -84,143 +148,53 @@ conf=r.getConfiguration() conf.attributes("docker").set("forceBasicAuth", false) repositoryManager.update(conf)' -usage () { - echo " This script is preparing Nexus data blob from docker images and npm packages" - echo " Usage:" - echo " ./$(basename $0) <config_file> [<target>]" - echo " " - echo " config_file is a file with defined variables, which are mandatory for this script" - echo " target is optional parameter where you can specify full path/name of resulted package" - echo " which replaces the value specified in configuration file" - echo " " - echo " Example: ./$(basename $0) ./package.conf /root/nexus_data.tar" - echo " " - echo " Parameters need to be defined in configuration file:" - echo " " - echo " NXS_SRC_DOCKER_IMG_DIR - directory of resource docker images" - echo " NXS_SRC_NPM_DIR - directory of resource npm packages" - echo " NXS_SRC_PYPI_DIR - directory of resource pypi packages" - echo " NXS_DOCKER_IMG_LIST - list of docker images to be pushed to Nexus repository" - echo " NXS_DOCKER_WO_LIST - list of docker images which uses default repository" - echo " NXS_NPM_LIST - list of npm packages to be published to Nexus repository" - echo " NXS_PYPI_LIST - list of pypi packages to be uploaded to Nexus repository" - echo " NEXUS_DATA_TAR - target tarball of Nexus data path/name" - echo " NEXUS_DATA_DIR - directory used for the Nexus blob build" - echo " NEXUS_IMAGE - Sonatype/Nexus3 docker image which will be used for data blob creation" - exit 1 -} - +# Prepare the Nexus configuration +NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}') ################################# # Prepare the local environment # ################################# -# Load the config file -if [ "${1}" == "-h" ] || [ -z "${1}" ]; then - usage -elif [ -f ${1} ]; then - . ${1} -else - echo "Missing mandatory configuration file!" - usage - exit 1 -fi - -if [ -n "${2}" ]; then - NEXUS_DATA_TAR="${2}" -fi - -for VAR in NXS_SRC_DOCKER_IMG_DIR NXS_SRC_NPM_DIR NXS_SRC_PYPI_DIR NXS_DOCKER_IMG_LIST NXS_DOCKER_WO_LIST NXS_NPM_LIST NXS_PYPI_LIST NEXUS_DATA_TAR NEXUS_DATA_DIR NEXUS_IMAGE; do - if [ -n "${!VAR}" ] ; then - echo "${VAR} is set to ${!VAR}" - else - echo "${VAR} is not set and it is mandatory" - FAIL="1" - fi -done - -if [ "${FAIL}" == "1" ]; then - echo "One or more mandatory variables are not set" - exit 1 -fi - -# Check the dependencies in the beginning - -# Install jq -if yum list installed "jq" >/dev/null 2>&1; then - echo "jq is already installed" -else - yum install -y --setopt=skip_missing_names_on_install=False http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/j/jq-1.5-1.el7.x86_64.rpm -fi - -# Install curl if necessary -if yum list installed "curl" >/dev/null 2>&1; then - echo "curl is already installed" -else - yum install -y --setopt=skip_missing_names_on_install=False curl -fi - -# Install expect if necessary -if yum list installed "expect" >/dev/null 2>&1; then - echo "expect is already installed" -else - yum install -y --setopt=skip_missing_names_on_install=False expect -fi - -# Install Docker (docker-ce in version 17.03 for RHEL) from online repositories if no version installed -if yum list installed "docker-ce" >/dev/null 2>&1 || which docker>/dev/null 2>&1; then - echo "Docker is already installed" -else - curl https://releases.rancher.com/install-docker/17.03.sh | sh -fi - -# Prepare the Nexus configuration -NEXUS_CONFIG=$(echo "${NEXUS_CONFIG_GROOVY}" | jq -Rsc '{"name":"configure", "type":"groovy", "content":.}') - # Add simulated domain names to /etc/hosts -cp /etc/hosts /etc/$(date +"%Y-%m-%d_%H-%M-%S")_hosts.bk +HOSTS_BACKUP="$(eval ${TIMESTAMP}_hosts.bk)" +cp /etc/hosts "/etc/${HOSTS_BACKUP}" for DNS in ${SIMUL_HOSTS}; do echo "127.0.0.1 ${DNS}" >> /etc/hosts done # Backup the current docker registry settings -if [ -f /root/.docker/config.json ]; then - mv /root/.docker/config.json /root/.docker/$(date +"%Y-%m-%d_%H-%M-%S")config.json.bk +if [ -f ~/.docker/config.json ]; then + DOCKER_CONF_BACKUP="$(eval ${TIMESTAMP}_config.json.bk)" + mv ~/.docker/config.json "~/.docker/${DOCKER_CONF_BACKUP}" fi ################################# # Docker repository preparation # ################################# -# Load all necessary images -for ARCHIVE in $(sed $'s/\r// ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${NXS_DOCKER_IMG_LIST} | awk '{ print $1 }'); do - docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE} -done - -for ARCHIVE in $(sed $'s/\r// ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${NXS_DOCKER_WO_LIST} | awk '{ print $1 }'); do - docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE} -done +# Load predefined Nexus image +docker load -i ${NEXUS_IMAGE_TAR} -# Tag docker images from default repository to simulated repository to be able to upload it to our private registry -for IMAGE in $(sed $'s/\r//' ${NXS_DOCKER_WO_LIST} | awk '{ print $1 }'); do - docker tag ${IMAGE} ${DOCKER_REGISTRY}/${IMAGE} +# Load all necessary images +for ARCHIVE in $(sed $'s/\r// ; /^#/d ; s/\:/\_/g ; s/\//\_/g ; s/$/\.tar/g' ${NXS_DOCKER_IMG_LIST} | awk '{ print $1 }'); do + docker load -i ${NXS_SRC_DOCKER_IMG_DIR}/${ARCHIVE} done - ################################ # Nexus repository preparation # ################################ -# Load predefined Nexus image -docker load -i ${NEXUS_IMAGE} - # Prepare nexus-data directory if [ -d ${NEXUS_DATA_DIR} ]; then - if [ "$(docker ps -q -f name=nexus)" ]; then - docker rm -f $(docker ps -aq -f name=nexus) - fi - cd ${NEXUS_DATA_DIR}/.. - mv ${NEXUS_DATA_DIR} $(date +"%Y-%m-%d_%H-%M-%S")_$(basename ${NEXUS_DATA_DIR})_bk + if [ "$(docker ps -q -f name="${NEXUS_DOMAIN}")" ]; then + echo "Removing container ${NEXUS_DOMAIN}" + docker rm -f $(docker ps -aq -f name="${NEXUS_DOMAIN}") + fi + pushd ${NEXUS_DATA_DIR}/.. + NXS_BACKUP="$(eval ${TIMESTAMP})_$(basename ${NEXUS_DATA_DIR})_bk" + mv ${NEXUS_DATA_DIR} "${NXS_BACKUP}" + echo "${NEXUS_DATA_DIR} already exists - backing up to ${NXS_BACKUP}" + popd fi mkdir -p ${NEXUS_DATA_DIR} @@ -231,24 +205,25 @@ chmod 777 ${NEXUS_DATA_DIR} docker images --no-trunc | grep sonatype/nexus3 | awk '{ print $1":"$2" "$3}' > ${NEXUS_DATA_DIR}/nexus.ver # Start the Nexus -NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name nexus -p 8081:8081 -p 8082:8082 -p 80:8082 -p 10001:8082 sonatype/nexus3) +NEXUS_CONT_ID=$(docker run -d --rm -v ${NEXUS_DATA_DIR}:/nexus-data:rw --name ${NEXUS_DOMAIN} ${PUBLISHED_PORTS} ${NEXUS_IMAGE}) echo "Waiting for Nexus to fully start" -until curl -su admin:admin123 http://${NEXUS_DOMAIN}:8081/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do +until curl -su ${NEXUS_USERNAME}:${NEXUS_PASSWORD} http://${NEXUS_DOMAIN}:${NEXUS_PORT}/service/metrics/healthcheck | grep '"healthy":true' > /dev/null ; do printf "." sleep 3 done echo -e "\nNexus started" # Configure the nexus repository -curl -X POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://admin:admin123@${NEXUS_DOMAIN}:8081/service/rest/v1/script -curl -X POST --header "Content-Type: text/plain" http://admin:admin123@${NEXUS_DOMAIN}:8081/service/rest/v1/script/configure/run +curl -sX POST --header 'Content-Type: application/json' --data-binary "${NEXUS_CONFIG}" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script +curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEXUS_PASSWORD}@${NEXUS_DOMAIN}:${NEXUS_PORT}/service/rest/v1/script/configure/run > /dev/null ########################### # Populate NPM repository # ########################### # Configure NPM registry to our Nexus repository -npm config set registry ${NPM_REGISTRY} +echo "Configure NPM registry to ${NPM_REGISTRY}" +npm config set registry "${NPM_REGISTRY}" # Login to NPM registry /usr/bin/expect <<EOF @@ -264,61 +239,86 @@ EOF # Patch problematic package pushd ${NXS_SRC_NPM_DIR} -tar xvzf tsscmp-1.0.5.tgz -rm -f tsscmp-1.0.5.tgz -sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json -tar -zcvf tsscmp-1.0.5.tgz package -rm -rf package +PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')" +if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then + tar xzf "${PATCHED_NPM}" + rm -f "${PATCHED_NPM}" + sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json + tar -zcf "${PATCHED_NPM}" package + rm -rf package +fi # Push NPM packages to Nexus repository -for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${NXS_NPM_LIST} | awk '{ print $1 }'); do - npm publish --access public ${ARCHIVE} +for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${NXS_NPM_LIST});do + npm publish --access public ${ARCHIVE} > /dev/null + echo "NPM ${ARCHIVE} pushed to Nexus" done popd -############################## -# Populate PyPi repository # -############################## +############################### +## Populate PyPi repository # +############################### pushd ${NXS_SRC_PYPI_DIR} for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do - twine upload -u ${NEXUS_USERNAME} -p ${NEXUS_PASSWORD} --repository-url ${PYPI_REGISTRY} ./${PACKAGE}* + twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* + echo "PYPI ${PACKAGE} pushed to Nexus" done popd -############################## -# Populate Docker repository # -############################## - -for REGISTRY in $(sed 's/\/.*//' ${NXS_DOCKER_IMG_LIST} | uniq) ${NEXUS_DOMAIN}:8082; do - docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" ${REGISTRY} > /dev/null -done +############################### +## Populate Docker repository # +############################### -for IMAGE in $(sed $'s/\r//' ${NXS_DOCKER_WO_LIST} | awk '{ print $1 }'); do - docker push ${DOCKER_REGISTRY}/${IMAGE} +# Login to simulated docker registries +for REGISTRY in $(sed -n '/\.[^/].*\//p' ${NXS_DOCKER_IMG_LIST} | sed -e 's/\/.*$//' | sort -u | grep -v ${DEFAULT_REGISTRY}) ${DOCKER_REGISTRY}; do + echo "Docker login to ${REGISTRY}" + docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" ${REGISTRY} > /dev/null done -for IMAGE in $(sed $'s/\r//' ${NXS_DOCKER_IMG_LIST} | awk '{ print $1 }'); do - docker push ${IMAGE} +# Push images to private nexus based on the list +# Images from default registry need to be tagged to private registry +# and those without defined repository in tag uses default repository 'library' +for IMAGE in $(sed $'s/\r// ; /^#/d' ${NXS_DOCKER_IMG_LIST} | awk '{ print $1 }'); do + PUSH="" + if [[ ${IMAGE} != *"/"* ]]; then + PUSH="${DOCKER_REGISTRY}/library/${IMAGE}" + elif [[ ${IMAGE} == *"${DEFAULT_REGISTRY}"* ]]; then + if [[ ${IMAGE} == *"/"*"/"* ]]; then + PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'/' <<< ${IMAGE})" + else + PUSH="$(sed 's/'"${DEFAULT_REGISTRY}"'/'"${DOCKER_REGISTRY}"'\/library/' <<< ${IMAGE})" + fi + elif [[ -z $(sed -n '/\.[^/].*\//p' <<< ${IMAGE}) ]]; then + PUSH="${DOCKER_REGISTRY}/${IMAGE}" + fi + if [[ ! -z ${PUSH} ]]; then + docker tag ${IMAGE} ${PUSH} + else + PUSH="${IMAGE}" + fi + docker push ${PUSH} + echo "${IMAGE} pushed as ${PUSH} to Nexus" done ############################## # Stop the Nexus and cleanup # ############################## +echo "Stopping Nexus and returning backups" + # Stop the Nexus -docker stop ${NEXUS_CONT_ID} +docker stop ${NEXUS_CONT_ID} > /dev/null -# Create the nexus-data package -cd ${NEXUS_DATA_DIR}/.. -echo "Packing the ${NEXUS_DATA_DIR} dir" -until tar -cf ${NEXUS_DATA_TAR} $(basename ${NEXUS_DATA_DIR}); do - printf "." - sleep 5 -done -echo "${NEXUS_DATA_TAR} has been created" +# Return backed up configuration files +mv -f "${HOSTS_BACKUP}" /etc/hosts + +if [ -f "~/.docker/${DOCKER_CONF_BACKUP}" ]; then + mv -f "${DOCKER_CONF_BACKUP}" ~/.docker/config.json +fi -# Return the previous version of /etc/hosts back to its place -mv -f $(ls -tr /etc/*hosts.bk | tail -1) /etc/hosts +# Return default settings +npm config set registry "https://registry.npmjs.org" +echo "Nexus blob is built" exit 0 diff --git a/build/common-functions.sh b/build/common-functions.sh index e39c4778..04ea2017 100755 --- a/build/common-functions.sh +++ b/build/common-functions.sh @@ -90,3 +90,9 @@ retry() { done } +clean_list() { + sed -e 's/\s*#.*$//' \ + -e '/^\s*$/d' ${1} | + tr -d '\r' | + awk '{ print $1 }' +} diff --git a/build/creating_data/download-docker-images.sh b/build/creating_data/download-docker-images.sh index 121cd5af..c0a0bed1 100755 --- a/build/creating_data/download-docker-images.sh +++ b/build/creating_data/download-docker-images.sh @@ -19,35 +19,21 @@ # COPYRIGHT NOTICE ENDS HERE -# boilerplate -RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' -if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then - SCRIPT_DIR=$(dirname "${0}") - LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") - . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh -fi +# Load common-functions library +. $(dirname ${0})/../common-functions.sh -SRC_IMAGE_LIST=$1 -if [[ -z "$SRC_IMAGE_LIST" ]]; then - SRC_IMAGE_LIST="docker_image_list.txt" +LIST_FILE="${1}" +if [[ -z "$LIST_FILE" ]]; then + LIST_FILE="docker_image_list.txt" fi echo "Download all images" -lines=$(cat $SRC_IMAGE_LIST | wc -l) +lines=$(clean_list "$LIST_FILE" | wc -l) line=1 -while read -r image; do +for image in $(clean_list "$LIST_FILE"); do echo "== pkg #$line of $lines ==" - - name=$(echo $image|awk '{print $1}') - digest=$(echo $image|awk '{print $2}') - - echo "$name digest:$digest" - if [[ "$digest" == "<none>" ]]; then - retry docker -l error pull "$name" - else - retry docker -l error pull "$name" - fi + echo "$image" + retry docker -l error pull "$image" line=$((line+1)) - -done < "$SRC_IMAGE_LIST" +done diff --git a/build/creating_data/download-files.sh b/build/creating_data/download-files.sh index 89e2026c..f687fda1 100755 --- a/build/creating_data/download-files.sh +++ b/build/creating_data/download-files.sh @@ -16,7 +16,11 @@ # # COPYRIGHT NOTICE ENDS HERE -LIST_FILE="$1" + +# Load common-functions library +. $(dirname ${0})/../common-functions.sh + +LIST_FILE="${1}" if [[ -z "$LIST_FILE" ]]; then echo "Missing list file" exit 1 @@ -28,13 +32,13 @@ if [[ -z "$outdir" ]]; then exit 1 fi -lines=$(cat "$LIST_FILE" | wc -l) +lines=$(clean_list "$LIST_FILE" | wc -l) cnt=1 # create output dir if not exists mkdir -p "$outdir" -while read -r line; do +for line in $(clean_list "$LIST_FILE"); do # www.springframework.org/schema/tool/spring-tool-4.3.xsd file="${line%%\?*}" filename=$(basename "$file") @@ -43,5 +47,4 @@ while read -r line; do # drop below 10b/10s curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$filename" &>/dev/null cnt=$((cnt+1)) - -done < "$LIST_FILE" +done
\ No newline at end of file diff --git a/build/creating_data/download-http-files.sh b/build/creating_data/download-http-files.sh index 06f41354..1144c66e 100755 --- a/build/creating_data/download-http-files.sh +++ b/build/creating_data/download-http-files.sh @@ -16,7 +16,11 @@ # # COPYRIGHT NOTICE ENDS HERE -LIST_FILE="$1" + +# Load common-functions library +. $(dirname ${0})/../common-functions.sh + +LIST_FILE="${1}" if [[ -z "$LIST_FILE" ]]; then echo "Missing list file" exit 1 @@ -28,13 +32,13 @@ if [[ -z "$outdir" ]]; then exit 1 fi -lines=$(cat "$LIST_FILE" | wc -l) +lines=$(clean_list "$LIST_FILE" | wc -l) cnt=1 # create output dir if not exists mkdir -p "$outdir" -while read -r line; do +for line in $(clean_list "$LIST_FILE"); do # www.springframework.org/schema/tool/spring-tool-4.3.xsd file="${line%%\?*}" echo "Downloading $cnt / $lines: $file" @@ -44,5 +48,4 @@ while read -r line; do # drop below 10b/10s curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null cnt=$((cnt+1)) - -done < "$LIST_FILE" +done
\ No newline at end of file diff --git a/build/creating_data/download-npm-pkgs.sh b/build/creating_data/download-npm-pkgs.sh index 9e8847c9..191dd5df 100755 --- a/build/creating_data/download-npm-pkgs.sh +++ b/build/creating_data/download-npm-pkgs.sh @@ -16,7 +16,10 @@ # # COPYRIGHT NOTICE ENDS HERE -LIST_FILE="$1" +# Load common-functions library +. $(dirname ${0})/../common-functions.sh + +LIST_FILE="${1}" if [[ -z "$LIST_FILE" ]]; then LIST_FILE="all_npm_list.txt" @@ -30,12 +33,10 @@ fi mkdir -p "$outdir" cd "$outdir" -lines=$(cat "$LIST_FILE" | wc -l) +lines=$(clean_list "$LIST_FILE" | wc -l) cnt=1 -while read -r line; do +for line in $(clean_list "$LIST_FILE"); do echo "== pkg #$cnt of $lines ==" - # yallist@2.1.2 npm pack $line cnt=$((cnt+1)) - -done < "$LIST_FILE" +done
\ No newline at end of file diff --git a/build/creating_data/download-pip.sh b/build/creating_data/download-pip.sh index dea60b52..fd4a4363 100755 --- a/build/creating_data/download-pip.sh +++ b/build/creating_data/download-pip.sh @@ -16,6 +16,10 @@ # # COPYRIGHT NOTICE ENDS HERE + +# Load common-functions library +. $(dirname ${0})/../common-functions.sh + LIST_FILE="$1" if [[ -z "$LIST_FILE" ]]; then echo "Missing list file" @@ -30,16 +34,15 @@ if [[ -z "$outdir" ]]; then exit 1 fi -lines=$(cat "$LIST_FILE" | wc -l) +lines=$(clean_list "$LIST_FILE" | wc -l) cnt=1 # create output dir if not exists mkdir -p "$outdir" cd "$outdir" -while read -r line; do +for line in $(clean_list "$LIST_FILE)"; do echo "Downloading $cnt / $lines: $line" pip download $line cnt=$((cnt+1)) - -done < "$LIST_FILE" +done diff --git a/build/creating_data/save-docker-images.sh b/build/creating_data/save-docker-images.sh index 4c764556..0a72d15b 100755 --- a/build/creating_data/save-docker-images.sh +++ b/build/creating_data/save-docker-images.sh @@ -19,15 +19,10 @@ # COPYRIGHT NOTICE ENDS HERE -# boilerplate -RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' -if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then - SCRIPT_DIR=$(dirname "${0}") - LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") - . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh -fi +# Load common-functions library +. $(dirname ${0})/../common-functions.sh -LIST="${1}" +LIST_FILE="${1}" IMG_DIR="${2}" if [[ -z "$IMG_DIR" ]]; then @@ -56,11 +51,9 @@ save_image() { echo "Save all images" line=1 -lines=$(wc -l ${LIST}) -while read -r image; do +lines=$(clean_list "$LIST_FILE" | wc -l) +for image in $(clean_list "$LIST_FILE"); do echo "== pkg #$line of $lines ==" - save_image "${image}" line=$((line+1)) - -done < "${LIST}" +done
\ No newline at end of file diff --git a/docs/BuildGuide.rst b/docs/BuildGuide.rst index a5a53485..a37b9781 100644 --- a/docs/BuildGuide.rst +++ b/docs/BuildGuide.rst @@ -234,82 +234,22 @@ Prerequisites: .. note:: In case you skipped the Part 2 for the artifacts download, please ensure that the copy of resources data are untarred in *./onap-offline/../resources/* -Whole nexus blob data tarball will be created by running script -build\_nexus\_blob.sh. It will load the listed docker images, run the -Nexus, configure it as npm and docker repository. Then it will push all -listed npm packages and docker images to the repositories. After all is -done the repository container is stopped and from the nexus-data -directory is created tarball. - -There are mandatory parameters need to be set in configuration file: - -+------------------------------+------------------------------------------------------------------------------------------+ -| Parameter | Description | -+==============================+==========================================================================================+ -| NXS\_SRC\_DOCKER\_IMG\_DIR | resource directory of docker images | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_SRC\_NPM\_DIR | resource directory of npm packages | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_SRC\_PYPI\_DIR | resource directory of npm packages | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_DOCKER\_IMG\_LIST | list of docker images to be pushed to Nexus repository | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_DOCKER\_WO\_LIST | list of docker images which uses default repository | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_NPM\_LIST | list of npm packages to be published to Nexus repository | -+------------------------------+------------------------------------------------------------------------------------------+ -| NXS\_PYPI\_LIST | list of pypi packages to be published to Nexus repository | -+------------------------------+------------------------------------------------------------------------------------------+ -| NEXUS\_DATA\_TAR | target tarball of Nexus data path/name | -+------------------------------+------------------------------------------------------------------------------------------+ -| NEXUS\_DATA\_DIR | directory used for the Nexus blob build | -+------------------------------+------------------------------------------------------------------------------------------+ -| NEXUS\_IMAGE | Sonatype/Nexus3 docker image which will be used for data blob creation for this script | -+------------------------------+------------------------------------------------------------------------------------------+ - -Some of the docker images using default registry requires special -treatment (e.g. they use different ports or SSL connection), therefore -there is the list NXS\_DOCKER\_WO\_LIST by which are the images retagged -to be able to push them to our nexus repository. -Following steps can be used to split *docker_images.list* into files for -NXS_DOCKER_IMG_LIST & NXS_DOCKER_WO_LIST variables. - -e.g. +Whole nexus blob data will be created by running script build\_nexus\_blob.sh. +It will load the listed docker images, run the Nexus, configure it as npm, pypi +and docker repositories. Then it will push all listed npm and pypi packages and +docker images to the repositories. After all is done the repository container +is stopped. -:: - - sed -n '/\.[^/].*\//p' onap_3.0.2-docker_images.list > /tmp/onap-me-data_lists/docker_img.list - sed -n '/\.[^/].*\//!p' onap_3.0.2-docker_images.list > /tmp/onap-me-data_lists/docker_no_registry.list - -.. note:: It's recomended to use abolute paths in the configuration file for the current script - -Example of the configuration file: - -:: - - NXS_SRC_DOCKER_IMG_DIR="/tmp/onap-offline/resources/offline_data/docker_images_for_nexus" - NXS_SRC_NPM_DIR="/tmp/onap-offline/resources/offline_data/npm_tar" - NXS_SRC_PYPI_DIR="/tmp/onap-offline/resources/offline_data/pypi" - NXS_DOCKER_IMG_LIST="/tmp/onap-me-data_lists/docker_img.list" - NXS_DOCKER_WO_LIST="/tmp/onap-me-data_lists/docker_no_registry.list" - NXS_NPM_LIST="/tmp/onap-offline/bash/tools/data_list/onap_3.0.x-npm_list.txt" - NXS_PYPI_LIST="/tmp/onap-offline/bash/tools/data_list/onap_3.0.x-pypi.list" - NEXUS_DATA_TAR="/root/nexus_data.tar" - NEXUS_DATA_DIR="/tmp/onap-offline/resources/nexus_data" - NEXUS_IMAGE="/tmp/onap-offline/resources/offline_data/docker_images_infra/sonatype_nexus3_3.15.2.tar" - -Once everything is ready you can run the script as following example: - -``$ ./onap-offline/build/build_nexus_blob.sh /root/nexus_build.conf`` +You can run the script as following example: -Where the nexus\_build.conf is the configuration file and the -/root/nexus\_data.tar is the destination tarball +``$ ./install/onap-offline/build_nexus_blob.sh onap_3.0.2`` -.. note:: Move, link or mount the NEXUS\_DATA\_DIR to the resources directory if there was different directory specified in configuration or use the resulting nexus\_data.tar for movement between machines. +Where the onap_3.0.2 is the tag to specify which lists will be used for the +resources -Once the Nexus data blob is created, the docker images and npm packages -can be deleted to reduce the package size as they won't be needed in the -installation time: +Once the Nexus data blob is created, the docker images and npm and pypi +packages can be deleted to reduce the package size as they won't be needed in +the installation time: E.g. |