Merge "Remove NPMs and PYPIs from nexus blob"

3 files changed, 76 insertions, 381 deletions

diff --git a/build/build_nexus_blob.sh b/build/build_nexus_blob.sh
index f964bf9d..1616386d 100755
--- a/build/build_nexus_blob.sh
+++ b/build/build_nexus_blob.sh
@@ -54,12 +54,14 @@ LOCAL_PATH="$(readlink -f $(dirname ${0}))"
 
 # Defaults
 DOCKER_LOAD="false"
+NPM_PUSH="false"
+PYPI_PUSH="false"
 DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
 NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
 LISTS_DIR="${LOCAL_PATH}/data_lists"
 
 # Required dependencies
-COMMANDS=(jq docker expect npm twine)
+COMMANDS=(jq docker)
 
 usage () {
     echo "
@@ -129,6 +131,37 @@ load_docker_images () {
     done
 }
 
+prepare_npm () {
+    # Configure NPM registry to our Nexus repository
+    echo "Configure NPM registry to ${NPM_REGISTRY}"
+    npm config set registry "${NPM_REGISTRY}"
+
+    # Login to NPM registry
+    /usr/bin/expect <<- EOF
+	spawn npm login
+	expect "Username:"
+	send "${NEXUS_USERNAME}\n"
+	expect "Password:"
+	send "${NEXUS_PASSWORD}\n"
+	expect Email:
+	send "${NEXUS_EMAIL}\n"
+	expect eof
+	EOF
+}
+
+patch_npm () {
+    # Patch problematic package
+    PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
+    if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null
+    then
+        tar xzf "${PATCHED_NPM}"
+        rm -f "${PATCHED_NPM}"
+        sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
+        tar -zcf "${PATCHED_NPM}" package
+        rm -rf package
+    fi
+}
+
 push_npm () {
     for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
         npm publish --access public ${ARCHIVE} > /dev/null
@@ -137,8 +170,8 @@ push_npm () {
 }
 
 push_pip () {
-    for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
-        twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
+    for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do
+        twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null
         echo "PYPI ${PACKAGE} pushed to Nexus"
     done
 }
@@ -176,19 +209,6 @@ push_docker () {
     done
 }
 
-# Verify all dependencies are available in PATH
-FAILED_COMMANDS=()
-for cmd in ${COMMANDS[*]}; do
-    command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
-done
-
-if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
-    echo "Following commands where not found in PATH and are required:"
-    echo ${FAILED_COMMANDS[*]}
-    echo "Aborting."
-    exit 1
-fi
-
 while [ "${1}" != "" ]; do
     case ${1} in
         -d | --docker )                    shift
@@ -199,13 +219,17 @@ while [ "${1}" != "" ]; do
                                            ;;
         -ld | --load-docker-images )       DOCKER_LOAD="true"
                                            ;;
-        -n | --npm )                       shift
+        -n | --npm )                       NPM_PUSH="true"
+                                           COMMANDS+=(expect npm)
+                                           shift
                                            NXS_NPM_LISTS+=("${1}")
                                            ;;
         -o | --output-directory )          shift
                                            NEXUS_DATA_DIR="${1}"
                                            ;;
-        -p | --pypi )                      shift
+        -p | --pypi )                      PYPI_PUSH="true"
+                                           COMMANDS+=(twine)
+                                           shift
                                            NXS_PYPI_LISTS+=("${1}")
                                            ;;
         -rl | --resource-list-directory )  shift
@@ -218,6 +242,19 @@ while [ "${1}" != "" ]; do
     shift
 done
 
+# Verify all dependencies are available in PATH
+FAILED_COMMANDS=()
+for cmd in ${COMMANDS[*]}; do
+    command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
+done
+
+if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
+    echo "Following commands where not found in PATH and are required:"
+    echo ${FAILED_COMMANDS[*]}
+    echo "Aborting."
+    exit 1
+fi
+
 # Setup directories with resources for docker, npm and pypi
 NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
 NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
@@ -227,18 +264,14 @@ NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
 NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
 NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
 NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
-NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
-NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"
 
 # Setup Nexus image used for build and install infra
 NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
 NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
 
 # Set default lists if nothing specific defined by user
-if [ $((${#NXS_DOCKER_IMG_LISTS[@]} + ${#NXS_NPM_LISTS[@]} + ${#NXS_PYPI_LISTS[@]})) -eq 0 ]; then
+if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then
     NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}")
-    NXS_NPM_LISTS[0]="${NXS_NPM_LIST}"
-    NXS_PYPI_LISTS[0]="${NXS_PYPI_LIST}"
 fi
 
 # Backup /etc/hosts
@@ -357,57 +390,36 @@ curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEX
 ###########################
 # Populate NPM repository #
 ###########################
-
-# Configure NPM registry to our Nexus repository
-echo "Configure NPM registry to ${NPM_REGISTRY}"
-npm config set registry "${NPM_REGISTRY}"
-
-# Login to NPM registry
-/usr/bin/expect <<EOF
-spawn npm login
-expect "Username:"
-send "${NEXUS_USERNAME}\n"
-expect "Password:"
-send "${NEXUS_PASSWORD}\n"
-expect Email:
-send "${NEXUS_EMAIL}\n"
-expect eof
-EOF
-
-# Patch problematic package
-pushd ${NXS_SRC_NPM_DIR}
-PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
-if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
-    tar xzf "${PATCHED_NPM}"
-    rm -f "${PATCHED_NPM}"
-    sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
-    tar -zcf "${PATCHED_NPM}" package
-    rm -rf package
+if [ $NPM_PUSH == "true" ]; then
+    prepare_npm
+    pushd ${NXS_SRC_NPM_DIR}
+    for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
+        patch_npm "${NPM_LIST}"
+        push_npm "${NPM_LIST}"
+    done
+    popd
+    # Return default settings
+    npm logout
+    npm config set registry "https://registry.npmjs.org"
 fi
 
-# Push NPM packages to Nexus repository
-for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
-    push_npm "${NPM_LIST}"
-done
-popd
-npm logout
-
 ###############################
 ##  Populate PyPi repository  #
 ###############################
-
-pushd ${NXS_SRC_PYPI_DIR}
-for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
-    push_pip "${PYPI_LIST}"
-done
-popd
+if [ $PYPI_PUSH == "true" ]; then
+    pushd ${NXS_SRC_PYPI_DIR}
+    for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
+        push_pip "${PYPI_LIST}"
+    done
+    popd
+fi
 
 ###############################
 ## Populate Docker repository #
 ###############################
 
 # Login to simulated docker registries
-# Push images to private nexus based on the list
+# Push images to private nexus based on the lists
 # Images from default registry need to be tagged to private registry
 # and those without defined repository in tag uses default repository 'library'
 for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
@@ -431,8 +443,5 @@ if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
     mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
 fi
 
-# Return default settings
-npm config set registry "https://registry.npmjs.org"
-
 echo "Nexus blob is built"
-exit 0
+exit 0
\ No newline at end of file
diff --git a/build/data_lists/onap_npm.list b/build/data_lists/onap_npm.list
deleted file mode 100644
index 058307e3..00000000
--- a/build/data_lists/onap_npm.list
+++ /dev/null
@@ -1,303 +0,0 @@
-async@0.1.22
-compression@1.1.2
-debug@0.7.4
-errorhandler@1.2.4
-fresh@0.2.4
-grunt-legacy-util@0.2.0
-js-yaml@2.0.5
-mime-types@2.0.14
-pause@0.0.1
-smtp-connection@1.3.8
-thenify@3.3.0
-underscore.string@2.3.3
-any-promise@1.3.0
-etag@1.4.0
-hoek@0.9.1
-method-override@2.2.0
-needle@0.9.2
-oauth@0.9.12
-path-is-absolute@1.0.1
-sentiment@0.2.3
-twitter-ng@0.6.2
-cookie-parser@1.3.5
-glob@3.1.21
-media-typer@0.3.0
-multer@1.1.0
-concat-map@0.0.1
-entities@1.1.2
-irc@0.3.7
-libqp@0.1.1
-mysql@2.7.0
-once@1.4.0
-qs@0.6.6
-serve-favicon@2.1.7
-utils-merge@1.0.0
-cookie@0.1.2
-debug@2.6.9
-hooker@0.2.3
-json-stringify-safe@5.0.1
-reduce-component@1.0.1
-underscore@1.7.0
-escape-html@1.0.1
-forever-agent@0.5.2
-lodash@2.4.2
-nopt@1.0.10
-qs@1.2.2
-cookie@0.1.3
-form-data@0.1.4
-grunt-legacy-log-utils@0.1.1
-libbase64@0.1.0
-qs@2.2.3
-glob@7.1.3
-nopt@3.0.1
-qs@2.2.4
-scmp@1.0.0
-type-is@1.5.7
-abbrev@1.1.1
-etag@1.5.1
-iconv-lite@0.4.24
-readable-stream@1.0.27-1
-strip-json-comments@1.0.4
-utf7@1.0.0
-async@0.9.2
-fs-extra@0.11.1
-mime@1.2.11
-optimist@0.3.7
-sinon@1.10.3
-buffer-from@1.1.1
-destroy@1.0.3
-mime-db@1.37.0
-readable-stream@1.1.14
-superagent@0.18.0
-which@1.0.9
-batch@0.5.1
-colors@0.6.2
-cookie-signature@1.0.6
-should@4.0.4
-uglify-js@2.4.15
-base64-url@1.2.1
-clone@0.1.18
-hyperquest@1.3.0
-methods@1.1.0
-on-finished@2.1.0
-readable-stream@2.3.6
-bl@0.9.5
-connect@2.26.1
-depd@0.4.5
-express@3.17.2
-htmlparser2@3.7.3
-libmime@0.1.7
-morgan@1.3.2
-request@2.42.0
-sntp@0.2.4
-through@2.2.7
-coffee-script@1.3.3
-csrf@2.0.7
-exit@0.1.2
-minimatch@3.0.4
-nodemailer-direct-transport@1.1.0
-on-finished@2.3.0
-stream-counter@0.2.0
-type-is@1.6.16
-debug@2.0.0
-formatio@1.0.2
-grunt-contrib-jshint@0.10.0
-mime-db@1.12.0
-native-or-bluebird@1.1.2
-bignumber.js@2.0.7
-connect-timeout@1.3.0
-negotiator@0.4.9
-raw-body@1.3.0
-shelljs@0.3.0
-when@3.4.6
-cookie-signature@1.0.5
-diff@1.0.7
-iconv-lite@0.2.11
-lodash@1.3.1
-ctype@0.5.3
-esprima@1.0.4
-glob@3.2.3
-minimatch@0.3.0
-require-all@1.0.0
-cryptiles@0.2.2
-dom-serializer@0.0.1
-hawk@1.1.1
-nan@1.0.0
-ipaddr.js@0.1.2
-minimatch@1.0.0
-resolve@0.3.1
-send@0.9.2
-util-deprecate@1.0.2
-assert-plus@0.1.5
-dicer@0.2.5
-inherits@1.0.2
-thenify-all@1.6.0
-vary@1.0.1
-glob@3.2.11
-imap@0.8.13
-psl@1.1.31
-xmlbuilder@9.0.7
-bytes@1.0.0
-domhandler@2.3.0
-hyperquest@0.3.0
-ms@0.6.2
-tunnel-agent@0.4.3
-ws@0.4.32
-body-parser@1.8.4
-commander@1.3.2
-merge-descriptors@0.0.2
-underscore.string@2.2.1
-boom@0.4.2
-commander@2.0.0
-jade@0.26.3
-mkdirp@0.3.0
-source-map@0.1.34
-brace-expansion@1.1.11
-cookiejar@1.3.2
-domelementtype@1.1.3
-extend@1.2.1
-graceful-fs@2.0.3
-inflight@1.0.6
-ncp@0.6.0
-punycode@2.1.1
-send@0.9.3
-statuses@1.5.0
-tough-cookie@2.5.0
-wordwrap@0.0.3
-balanced-match@1.0.0
-combined-stream@0.0.7
-finalhandler@0.2.0
-mkdirp@0.3.5
-node-uuid@1.4.8
-options@0.0.6
-pretty-data@0.40.0
-string_decoder@1.1.1
-underscore.string@2.4.0
-fs.realpath@1.0.0
-grunt-simple-mocha@0.4.0
-mime-types@2.1.21
-busboy@0.2.14
-cors@2.4.2
-domhandler@2.2.1
-graceful-fs@4.1.15
-jshint@2.5.11
-minimatch@0.2.14
-nodemailer-smtp-transport@0.1.13
-samsam@1.1.3
-wrappy@1.0.2
-cron@1.0.4
-ee-first@1.1.1
-growl@1.8.1
-mocha@1.21.4
-response-time@2.0.1
-dateformat@1.0.2-1.2.3
-duplexer2@0.0.2
-eventemitter2@0.4.14
-multiparty@3.3.2
-sax@0.6.1
-compressible@2.0.15
-CSSwhat@0.4.7
-glob@5.0.15
-util@0.11.1
-concat-stream@1.6.2
-etag@1.3.1
-mustache@0.8.2
-rndm@1.1.1
-vhost@3.0.2
-async@0.2.10
-domelementtype@1.3.1
-isarray@1.0.0
-minimist@0.0.8
-on-headers@1.0.1
-uid-safe@1.0.1
-addressparser@0.3.2
-http-errors@1.2.8
-mkdirp@0.5.1
-caseless@0.6.0
-form-data@0.1.2
-inherits@2.0.3
-readable-stream@1.0.34
-tinycolor@0.0.1
-accepts@1.1.4
-cli@0.6.6
-typedarray@0.0.6
-argparse@0.1.16
-graceful-fs@1.2.3
-methods@1.0.0
-safe-buffer@5.1.2
-amdefine@1.0.1
-buildmail@1.3.0
-express-session@1.8.2
-jsonfile@2.4.0
-string_decoder@0.10.31
-commander@2.1.0
-csurf@1.6.6
-domutils@1.4.3
-formidable@1.0.14
-grunt-cli@0.1.13
-isarray@0.0.1
-object-assign@3.0.0
-range-parser@1.0.3
-sigmund@1.0.1
-uglify-to-browserify@1.0.2
-ansi-color@0.2.1
-basic-auth@1.0.0
-core-util-is@1.0.2
-entities@1.0.0
-follow-redirects@0.0.3
-ms@2.0.0
-parseurl@1.3.2
-rimraf@2.2.8
-domutils@1.5.1
-getobject@0.1.0
-htmlparser2@3.8.3
-libmime@1.2.0
-component-emitter@1.1.2
-duplexer@0.1.1
-keypress@0.1.0
-mqtt@0.3.13
-oauth-sign@0.4.0
-streamsearch@0.1.2
-CSSselect@0.4.1
-grunt-legacy-log@0.1.3
-rimraf@2.6.3
-nodemailer-wellknown@0.1.10
-console-browserify@1.1.0
-date-now@0.1.4
-grunt@0.4.5
-mime@1.2.5
-findup-sync@0.1.3
-lodash@3.7.0
-nodemailer@1.3.0
-process-nextick-args@2.0.0
-safer-buffer@2.1.2
-supertest@0.13.0
-xml2js@0.4.4
-ee-first@1.0.5
-http-signature@0.10.1
-lodash@0.9.2
-mkdirp@0.5.0
-stringstream@0.0.6
-underscore@1.6.0
-append-field@0.1.0
-iconv-lite@0.4.4
-basic-auth-connect@1.0.0
-commander@0.6.1
-methods@0.0.1
-mz@1.3.0
-serve-static@1.6.5
-uid-safe@1.1.0
-crc@3.0.0
-delayed-stream@0.0.5
-libqp@1.1.0
-underscore@1.9.1
-aws-sign2@0.5.0
-mime-types@1.0.2
-proxy-addr@1.0.1
-serve-index@1.2.1
-through2@0.6.5
-asn1@0.1.11
-cheerio@0.17.0
-crc@3.2.1
-lru-cache@2.7.3
-xtend@4.0.1
\ No newline at end of file
diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list
deleted file mode 100644
index 7d9f826f..00000000
--- a/build/data_lists/onap_pip_packages.list
+++ /dev/null
@@ -1,11 +0,0 @@
-Click==7.0
-certifi==2019.9.11
-chardet==3.0.4
-Flask==1.1.1
-idna==2.8
-itsdangerous==1.1.0
-Jinja2==2.10.1
-MarkupSafe==1.1.1
-requests==2.22.0
-urllib3==1.25.6
-Werkzeug==0.16.0