summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTomáš Levora <t.levora@partner.samsung.com>2019-09-04 13:56:53 +0200
committerTomáš Levora <t.levora@partner.samsung.com>2019-10-15 12:27:13 +0000
commitbe74884dc77ebed757df0093f9a5fb260d322196 (patch)
treea6b8b34eb52975612dfdd7a6aec3f4766123455f
parent1b402b0db23d65f9c6c716c803fa350839ff2f6b (diff)
Remove NPMs and PYPIs from nexus blob
Issue-ID: OOM-2015 Change-Id: Ie8bc1219291b52410ca7ec5a40ed3e70a11cd315 Signed-off-by: Tomáš Levora <t.levora@partner.samsung.com>
-rwxr-xr-xbuild/build_nexus_blob.sh143
-rw-r--r--build/data_lists/onap_npm.list303
-rw-r--r--build/data_lists/onap_pip_packages.list11
3 files changed, 76 insertions, 381 deletions
diff --git a/build/build_nexus_blob.sh b/build/build_nexus_blob.sh
index f964bf9d..1616386d 100755
--- a/build/build_nexus_blob.sh
+++ b/build/build_nexus_blob.sh
@@ -54,12 +54,14 @@ LOCAL_PATH="$(readlink -f $(dirname ${0}))"
# Defaults
DOCKER_LOAD="false"
+NPM_PUSH="false"
+PYPI_PUSH="false"
DATA_DIR="$(realpath ${LOCAL_PATH}/../../resources)"
NEXUS_DATA_DIR="${DATA_DIR}/nexus_data"
LISTS_DIR="${LOCAL_PATH}/data_lists"
# Required dependencies
-COMMANDS=(jq docker expect npm twine)
+COMMANDS=(jq docker)
usage () {
echo "
@@ -129,6 +131,37 @@ load_docker_images () {
done
}
+prepare_npm () {
+ # Configure NPM registry to our Nexus repository
+ echo "Configure NPM registry to ${NPM_REGISTRY}"
+ npm config set registry "${NPM_REGISTRY}"
+
+ # Login to NPM registry
+ /usr/bin/expect <<- EOF
+ spawn npm login
+ expect "Username:"
+ send "${NEXUS_USERNAME}\n"
+ expect "Password:"
+ send "${NEXUS_PASSWORD}\n"
+ expect Email:
+ send "${NEXUS_EMAIL}\n"
+ expect eof
+ EOF
+}
+
+patch_npm () {
+ # Patch problematic package
+ PATCHED_NPM="$(grep tsscmp ${1} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
+ if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null
+ then
+ tar xzf "${PATCHED_NPM}"
+ rm -f "${PATCHED_NPM}"
+ sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
+ tar -zcf "${PATCHED_NPM}" package
+ rm -rf package
+ fi
+}
+
push_npm () {
for ARCHIVE in $(sed $'s/\r// ; s/\\@/\-/g ; s/$/\.tgz/g' ${1}); do
npm publish --access public ${ARCHIVE} > /dev/null
@@ -137,8 +170,8 @@ push_npm () {
}
push_pip () {
- for PACKAGE in $(sed $'s/\r//; s/==/-/' ${NXS_PYPI_LIST}); do
- twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}*
+ for PACKAGE in $(sed $'s/\r//; s/==/-/' ${1}); do
+ twine upload -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" --repository-url ${PYPI_REGISTRY} ${PACKAGE}* > /dev/null
echo "PYPI ${PACKAGE} pushed to Nexus"
done
}
@@ -176,19 +209,6 @@ push_docker () {
done
}
-# Verify all dependencies are available in PATH
-FAILED_COMMANDS=()
-for cmd in ${COMMANDS[*]}; do
- command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
-done
-
-if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
- echo "Following commands where not found in PATH and are required:"
- echo ${FAILED_COMMANDS[*]}
- echo "Aborting."
- exit 1
-fi
-
while [ "${1}" != "" ]; do
case ${1} in
-d | --docker ) shift
@@ -199,13 +219,17 @@ while [ "${1}" != "" ]; do
;;
-ld | --load-docker-images ) DOCKER_LOAD="true"
;;
- -n | --npm ) shift
+ -n | --npm ) NPM_PUSH="true"
+ COMMANDS+=(expect npm)
+ shift
NXS_NPM_LISTS+=("${1}")
;;
-o | --output-directory ) shift
NEXUS_DATA_DIR="${1}"
;;
- -p | --pypi ) shift
+ -p | --pypi ) PYPI_PUSH="true"
+ COMMANDS+=(twine)
+ shift
NXS_PYPI_LISTS+=("${1}")
;;
-rl | --resource-list-directory ) shift
@@ -218,6 +242,19 @@ while [ "${1}" != "" ]; do
shift
done
+# Verify all dependencies are available in PATH
+FAILED_COMMANDS=()
+for cmd in ${COMMANDS[*]}; do
+ command -v $cmd >/dev/null 2>&1 || FAILED_COMMANDS+=($cmd)
+done
+
+if [ ${#FAILED_COMMANDS[*]} -gt 0 ]; then
+ echo "Following commands where not found in PATH and are required:"
+ echo ${FAILED_COMMANDS[*]}
+ echo "Aborting."
+ exit 1
+fi
+
# Setup directories with resources for docker, npm and pypi
NXS_SRC_DOCKER_IMG_DIR="${DATA_DIR}/offline_data/docker_images_for_nexus"
NXS_SRC_NPM_DIR="${DATA_DIR}/offline_data/npm_tar"
@@ -227,18 +264,14 @@ NXS_SRC_PYPI_DIR="${DATA_DIR}/offline_data/pypi"
NXS_INFRA_LIST="${LISTS_DIR}/infra_docker_images.list"
NXS_DOCKER_IMG_LIST="${LISTS_DIR}/onap_docker_images.list"
NXS_RKE_DOCKER_IMG_LIST="${LISTS_DIR}/rke_docker_images.list"
-NXS_NPM_LIST="${LISTS_DIR}/onap_npm.list"
-NXS_PYPI_LIST="${LISTS_DIR}/onap_pip_packages.list"
# Setup Nexus image used for build and install infra
NEXUS_IMAGE="$(grep sonatype/nexus3 ${NXS_INFRA_LIST})"
NEXUS_IMAGE_TAR="${DATA_DIR}/offline_data/docker_images_infra/$(sed 's/\//\_/ ; s/$/\.tar/ ; s/\:/\_/' <<< ${NEXUS_IMAGE})"
# Set default lists if nothing specific defined by user
-if [ $((${#NXS_DOCKER_IMG_LISTS[@]} + ${#NXS_NPM_LISTS[@]} + ${#NXS_PYPI_LISTS[@]})) -eq 0 ]; then
+if [ ${#NXS_DOCKER_IMG_LISTS[@]} -eq 0 ]; then
NXS_DOCKER_IMG_LISTS=("${NXS_DOCKER_IMG_LIST}" "${NXS_RKE_DOCKER_IMG_LIST}")
- NXS_NPM_LISTS[0]="${NXS_NPM_LIST}"
- NXS_PYPI_LISTS[0]="${NXS_PYPI_LIST}"
fi
# Backup /etc/hosts
@@ -357,57 +390,36 @@ curl -sX POST --header "Content-Type: text/plain" http://${NEXUS_USERNAME}:${NEX
###########################
# Populate NPM repository #
###########################
-
-# Configure NPM registry to our Nexus repository
-echo "Configure NPM registry to ${NPM_REGISTRY}"
-npm config set registry "${NPM_REGISTRY}"
-
-# Login to NPM registry
-/usr/bin/expect <<EOF
-spawn npm login
-expect "Username:"
-send "${NEXUS_USERNAME}\n"
-expect "Password:"
-send "${NEXUS_PASSWORD}\n"
-expect Email:
-send "${NEXUS_EMAIL}\n"
-expect eof
-EOF
-
-# Patch problematic package
-pushd ${NXS_SRC_NPM_DIR}
-PATCHED_NPM="$(grep tsscmp ${NXS_NPM_LIST} | sed $'s/\r// ; s/\\@/\-/ ; s/$/\.tgz/')"
-if [[ ! -z "${PATCHED_NPM}" ]] && ! zgrep -aq "${NPM_REGISTRY}" "${PATCHED_NPM}" 2>/dev/null; then
- tar xzf "${PATCHED_NPM}"
- rm -f "${PATCHED_NPM}"
- sed -i 's|\"registry\":\ \".*\"|\"registry\":\ \"'"${NPM_REGISTRY}"'\"|g' package/package.json
- tar -zcf "${PATCHED_NPM}" package
- rm -rf package
+if [ $NPM_PUSH == "true" ]; then
+ prepare_npm
+ pushd ${NXS_SRC_NPM_DIR}
+ for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
+ patch_npm "${NPM_LIST}"
+ push_npm "${NPM_LIST}"
+ done
+ popd
+ # Return default settings
+ npm logout
+ npm config set registry "https://registry.npmjs.org"
fi
-# Push NPM packages to Nexus repository
-for NPM_LIST in "${NXS_NPM_LISTS[@]}"; do
- push_npm "${NPM_LIST}"
-done
-popd
-npm logout
-
###############################
## Populate PyPi repository #
###############################
-
-pushd ${NXS_SRC_PYPI_DIR}
-for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
- push_pip "${PYPI_LIST}"
-done
-popd
+if [ $PYPI_PUSH == "true" ]; then
+ pushd ${NXS_SRC_PYPI_DIR}
+ for PYPI_LIST in "${NXS_PYPI_LISTS[@]}"; do
+ push_pip "${PYPI_LIST}"
+ done
+ popd
+fi
###############################
## Populate Docker repository #
###############################
# Login to simulated docker registries
-# Push images to private nexus based on the list
+# Push images to private nexus based on the lists
# Images from default registry need to be tagged to private registry
# and those without defined repository in tag uses default repository 'library'
for DOCKER_IMG_LIST in "${NXS_DOCKER_IMG_LISTS[@]}"; do
@@ -431,8 +443,5 @@ if [ -f ~/.docker/${DOCKER_CONF_BACKUP} ]; then
mv -f ~/.docker/${DOCKER_CONF_BACKUP} ~/.docker/config.json
fi
-# Return default settings
-npm config set registry "https://registry.npmjs.org"
-
echo "Nexus blob is built"
-exit 0
+exit 0 \ No newline at end of file
diff --git a/build/data_lists/onap_npm.list b/build/data_lists/onap_npm.list
deleted file mode 100644
index 058307e3..00000000
--- a/build/data_lists/onap_npm.list
+++ /dev/null
@@ -1,303 +0,0 @@
-async@0.1.22
-compression@1.1.2
-debug@0.7.4
-errorhandler@1.2.4
-fresh@0.2.4
-grunt-legacy-util@0.2.0
-js-yaml@2.0.5
-mime-types@2.0.14
-pause@0.0.1
-smtp-connection@1.3.8
-thenify@3.3.0
-underscore.string@2.3.3
-any-promise@1.3.0
-etag@1.4.0
-hoek@0.9.1
-method-override@2.2.0
-needle@0.9.2
-oauth@0.9.12
-path-is-absolute@1.0.1
-sentiment@0.2.3
-twitter-ng@0.6.2
-cookie-parser@1.3.5
-glob@3.1.21
-media-typer@0.3.0
-multer@1.1.0
-concat-map@0.0.1
-entities@1.1.2
-irc@0.3.7
-libqp@0.1.1
-mysql@2.7.0
-once@1.4.0
-qs@0.6.6
-serve-favicon@2.1.7
-utils-merge@1.0.0
-cookie@0.1.2
-debug@2.6.9
-hooker@0.2.3
-json-stringify-safe@5.0.1
-reduce-component@1.0.1
-underscore@1.7.0
-escape-html@1.0.1
-forever-agent@0.5.2
-lodash@2.4.2
-nopt@1.0.10
-qs@1.2.2
-cookie@0.1.3
-form-data@0.1.4
-grunt-legacy-log-utils@0.1.1
-libbase64@0.1.0
-qs@2.2.3
-glob@7.1.3
-nopt@3.0.1
-qs@2.2.4
-scmp@1.0.0
-type-is@1.5.7
-abbrev@1.1.1
-etag@1.5.1
-iconv-lite@0.4.24
-readable-stream@1.0.27-1
-strip-json-comments@1.0.4
-utf7@1.0.0
-async@0.9.2
-fs-extra@0.11.1
-mime@1.2.11
-optimist@0.3.7
-sinon@1.10.3
-buffer-from@1.1.1
-destroy@1.0.3
-mime-db@1.37.0
-readable-stream@1.1.14
-superagent@0.18.0
-which@1.0.9
-batch@0.5.1
-colors@0.6.2
-cookie-signature@1.0.6
-should@4.0.4
-uglify-js@2.4.15
-base64-url@1.2.1
-clone@0.1.18
-hyperquest@1.3.0
-methods@1.1.0
-on-finished@2.1.0
-readable-stream@2.3.6
-bl@0.9.5
-connect@2.26.1
-depd@0.4.5
-express@3.17.2
-htmlparser2@3.7.3
-libmime@0.1.7
-morgan@1.3.2
-request@2.42.0
-sntp@0.2.4
-through@2.2.7
-coffee-script@1.3.3
-csrf@2.0.7
-exit@0.1.2
-minimatch@3.0.4
-nodemailer-direct-transport@1.1.0
-on-finished@2.3.0
-stream-counter@0.2.0
-type-is@1.6.16
-debug@2.0.0
-formatio@1.0.2
-grunt-contrib-jshint@0.10.0
-mime-db@1.12.0
-native-or-bluebird@1.1.2
-bignumber.js@2.0.7
-connect-timeout@1.3.0
-negotiator@0.4.9
-raw-body@1.3.0
-shelljs@0.3.0
-when@3.4.6
-cookie-signature@1.0.5
-diff@1.0.7
-iconv-lite@0.2.11
-lodash@1.3.1
-ctype@0.5.3
-esprima@1.0.4
-glob@3.2.3
-minimatch@0.3.0
-require-all@1.0.0
-cryptiles@0.2.2
-dom-serializer@0.0.1
-hawk@1.1.1
-nan@1.0.0
-ipaddr.js@0.1.2
-minimatch@1.0.0
-resolve@0.3.1
-send@0.9.2
-util-deprecate@1.0.2
-assert-plus@0.1.5
-dicer@0.2.5
-inherits@1.0.2
-thenify-all@1.6.0
-vary@1.0.1
-glob@3.2.11
-imap@0.8.13
-psl@1.1.31
-xmlbuilder@9.0.7
-bytes@1.0.0
-domhandler@2.3.0
-hyperquest@0.3.0
-ms@0.6.2
-tunnel-agent@0.4.3
-ws@0.4.32
-body-parser@1.8.4
-commander@1.3.2
-merge-descriptors@0.0.2
-underscore.string@2.2.1
-boom@0.4.2
-commander@2.0.0
-jade@0.26.3
-mkdirp@0.3.0
-source-map@0.1.34
-brace-expansion@1.1.11
-cookiejar@1.3.2
-domelementtype@1.1.3
-extend@1.2.1
-graceful-fs@2.0.3
-inflight@1.0.6
-ncp@0.6.0
-punycode@2.1.1
-send@0.9.3
-statuses@1.5.0
-tough-cookie@2.5.0
-wordwrap@0.0.3
-balanced-match@1.0.0
-combined-stream@0.0.7
-finalhandler@0.2.0
-mkdirp@0.3.5
-node-uuid@1.4.8
-options@0.0.6
-pretty-data@0.40.0
-string_decoder@1.1.1
-underscore.string@2.4.0
-fs.realpath@1.0.0
-grunt-simple-mocha@0.4.0
-mime-types@2.1.21
-busboy@0.2.14
-cors@2.4.2
-domhandler@2.2.1
-graceful-fs@4.1.15
-jshint@2.5.11
-minimatch@0.2.14
-nodemailer-smtp-transport@0.1.13
-samsam@1.1.3
-wrappy@1.0.2
-cron@1.0.4
-ee-first@1.1.1
-growl@1.8.1
-mocha@1.21.4
-response-time@2.0.1
-dateformat@1.0.2-1.2.3
-duplexer2@0.0.2
-eventemitter2@0.4.14
-multiparty@3.3.2
-sax@0.6.1
-compressible@2.0.15
-CSSwhat@0.4.7
-glob@5.0.15
-util@0.11.1
-concat-stream@1.6.2
-etag@1.3.1
-mustache@0.8.2
-rndm@1.1.1
-vhost@3.0.2
-async@0.2.10
-domelementtype@1.3.1
-isarray@1.0.0
-minimist@0.0.8
-on-headers@1.0.1
-uid-safe@1.0.1
-addressparser@0.3.2
-http-errors@1.2.8
-mkdirp@0.5.1
-caseless@0.6.0
-form-data@0.1.2
-inherits@2.0.3
-readable-stream@1.0.34
-tinycolor@0.0.1
-accepts@1.1.4
-cli@0.6.6
-typedarray@0.0.6
-argparse@0.1.16
-graceful-fs@1.2.3
-methods@1.0.0
-safe-buffer@5.1.2
-amdefine@1.0.1
-buildmail@1.3.0
-express-session@1.8.2
-jsonfile@2.4.0
-string_decoder@0.10.31
-commander@2.1.0
-csurf@1.6.6
-domutils@1.4.3
-formidable@1.0.14
-grunt-cli@0.1.13
-isarray@0.0.1
-object-assign@3.0.0
-range-parser@1.0.3
-sigmund@1.0.1
-uglify-to-browserify@1.0.2
-ansi-color@0.2.1
-basic-auth@1.0.0
-core-util-is@1.0.2
-entities@1.0.0
-follow-redirects@0.0.3
-ms@2.0.0
-parseurl@1.3.2
-rimraf@2.2.8
-domutils@1.5.1
-getobject@0.1.0
-htmlparser2@3.8.3
-libmime@1.2.0
-component-emitter@1.1.2
-duplexer@0.1.1
-keypress@0.1.0
-mqtt@0.3.13
-oauth-sign@0.4.0
-streamsearch@0.1.2
-CSSselect@0.4.1
-grunt-legacy-log@0.1.3
-rimraf@2.6.3
-nodemailer-wellknown@0.1.10
-console-browserify@1.1.0
-date-now@0.1.4
-grunt@0.4.5
-mime@1.2.5
-findup-sync@0.1.3
-lodash@3.7.0
-nodemailer@1.3.0
-process-nextick-args@2.0.0
-safer-buffer@2.1.2
-supertest@0.13.0
-xml2js@0.4.4
-ee-first@1.0.5
-http-signature@0.10.1
-lodash@0.9.2
-mkdirp@0.5.0
-stringstream@0.0.6
-underscore@1.6.0
-append-field@0.1.0
-iconv-lite@0.4.4
-basic-auth-connect@1.0.0
-commander@0.6.1
-methods@0.0.1
-mz@1.3.0
-serve-static@1.6.5
-uid-safe@1.1.0
-crc@3.0.0
-delayed-stream@0.0.5
-libqp@1.1.0
-underscore@1.9.1
-aws-sign2@0.5.0
-mime-types@1.0.2
-proxy-addr@1.0.1
-serve-index@1.2.1
-through2@0.6.5
-asn1@0.1.11
-cheerio@0.17.0
-crc@3.2.1
-lru-cache@2.7.3
-xtend@4.0.1 \ No newline at end of file
diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list
deleted file mode 100644
index 7d9f826f..00000000
--- a/build/data_lists/onap_pip_packages.list
+++ /dev/null
@@ -1,11 +0,0 @@
-Click==7.0
-certifi==2019.9.11
-chardet==3.0.4
-Flask==1.1.1
-idna==2.8
-itsdangerous==1.1.0
-Jinja2==2.10.1
-MarkupSafe==1.1.1
-requests==2.22.0
-urllib3==1.25.6
-Werkzeug==0.16.0