diff options
Diffstat (limited to 'onap-offline/bash/tools/creating_data')
21 files changed, 913 insertions, 0 deletions
diff --git a/onap-offline/bash/tools/creating_data/create_nginx_image/01create-image.sh b/onap-offline/bash/tools/creating_data/create_nginx_image/01create-image.sh new file mode 100755 index 0000000..86d96cf --- /dev/null +++ b/onap-offline/bash/tools/creating_data/create_nginx_image/01create-image.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +script_dir="$(dirname ${BASH_SOURCE[0]})" +cd "$script_dir" +docker build -t own_nginx . diff --git a/onap-offline/bash/tools/creating_data/create_nginx_image/Dockerfile b/onap-offline/bash/tools/creating_data/create_nginx_image/Dockerfile new file mode 100644 index 0000000..412d4ed --- /dev/null +++ b/onap-offline/bash/tools/creating_data/create_nginx_image/Dockerfile @@ -0,0 +1,32 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +FROM nginx:alpine + +RUN apk add --no-cache --update fcgiwrap git git-daemon spawn-fcgi + +#RUN mkdir /srv/http +#VOLUME ["/var/log/nginx"] + +COPY gitconfig /etc/ +#COPY nginx.conf /etc/nginx/nginx.conf + +CMD spawn-fcgi -M 666 -s /var/run/fcgiwrap.socket /usr/bin/fcgiwrap && \ + nginx -g "daemon off;" + +#CMD ["spawn-fcgi","-M 666","-s /var/run/fcgiwrap.socket","/usr/sbin/fcgiwrap"] diff --git a/onap-offline/bash/tools/creating_data/create_nginx_image/gitconfig b/onap-offline/bash/tools/creating_data/create_nginx_image/gitconfig new file mode 100644 index 0000000..0c6ab11 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/create_nginx_image/gitconfig @@ -0,0 +1,2 @@ +[http] + postBuffer = 1048576 diff --git a/onap-offline/bash/tools/creating_data/docker-login.sh b/onap-offline/bash/tools/creating_data/docker-login.sh new file mode 100755 index 0000000..d57e23b --- /dev/null +++ b/onap-offline/bash/tools/creating_data/docker-login.sh @@ -0,0 +1,38 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +# boilerplate +RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' +if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then + SCRIPT_DIR=$(dirname "${0}") + LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") + . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh +fi + +echo "Reading configuration" +get_configuration + +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" docker.elastic.co +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" gcr.io +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" nexus3.onap.org:10001 +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" registry.hub.docker.com +docker login -u "${NEXUS_USERNAME}" -p "${NEXUS_PASSWORD}" "$NEXUS_FQDN" diff --git a/onap-offline/bash/tools/creating_data/download-bin-tools.sh b/onap-offline/bash/tools/creating_data/download-bin-tools.sh new file mode 100755 index 0000000..b48cd2d --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-bin-tools.sh @@ -0,0 +1,51 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +outdir="$1" +if [[ -z "$outdir" ]]; then + echo "Missing arg outdir" + exit 1 +fi + + +mkdir -p "$outdir" +cd "$outdir" + +download() { + url="$1" + url_file="${url%%\?*}" + file=$(basename "$url_file") + echo "Downloading $url" + curl --retry 5 -y 10 -Y 10 --location "$url" -o "$file" +} + +download "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64" +mv ./jq-linux64 ./jq + +download "https://storage.googleapis.com/kubernetes-release/release/v1.8.10/bin/linux/amd64/kubectl" + +download "https://storage.googleapis.com/kubernetes-helm/helm-v2.8.2-linux-amd64.tar.gz" +tar -xf ./helm-v2.8.2-linux-amd64.tar.gz linux-amd64/helm -O > helm +rm ./helm-v2.8.2-linux-amd64.tar.gz + +download "https://github.com/rancher/cli/releases/download/v0.6.7/rancher-linux-amd64-v0.6.7.tar.gz" +tar -xf ./rancher-linux-amd64-v0.6.7.tar.gz ./rancher-v0.6.7/rancher -O > rancher +rm ./rancher-linux-amd64-v0.6.7.tar.gz + + +chmod a+x ./helm ./jq ./kubectl ./rancher diff --git a/onap-offline/bash/tools/creating_data/download-docker-images.sh b/onap-offline/bash/tools/creating_data/download-docker-images.sh new file mode 100755 index 0000000..121cd5a --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-docker-images.sh @@ -0,0 +1,53 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +# boilerplate +RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' +if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then + SCRIPT_DIR=$(dirname "${0}") + LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") + . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh +fi + +SRC_IMAGE_LIST=$1 +if [[ -z "$SRC_IMAGE_LIST" ]]; then + SRC_IMAGE_LIST="docker_image_list.txt" +fi + +echo "Download all images" + +lines=$(cat $SRC_IMAGE_LIST | wc -l) +line=1 +while read -r image; do + echo "== pkg #$line of $lines ==" + + name=$(echo $image|awk '{print $1}') + digest=$(echo $image|awk '{print $2}') + + echo "$name digest:$digest" + if [[ "$digest" == "<none>" ]]; then + retry docker -l error pull "$name" + else + retry docker -l error pull "$name" + fi + line=$((line+1)) + +done < "$SRC_IMAGE_LIST" diff --git a/onap-offline/bash/tools/creating_data/download-files.sh b/onap-offline/bash/tools/creating_data/download-files.sh new file mode 100755 index 0000000..89e2026 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-files.sh @@ -0,0 +1,47 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +LIST_FILE="$1" +if [[ -z "$LIST_FILE" ]]; then + echo "Missing list file" + exit 1 +fi + +outdir="$2" +if [[ -z "$outdir" ]]; then + echo "Missing output directory" + exit 1 +fi + +lines=$(cat "$LIST_FILE" | wc -l) +cnt=1 + +# create output dir if not exists +mkdir -p "$outdir" + +while read -r line; do + # www.springframework.org/schema/tool/spring-tool-4.3.xsd + file="${line%%\?*}" + filename=$(basename "$file") + echo "Downloading $cnt / $lines: $file" + # following curl params are ensurring 5 reties and cut-off if connectivity will + # drop below 10b/10s + curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$filename" &>/dev/null + cnt=$((cnt+1)) + +done < "$LIST_FILE" diff --git a/onap-offline/bash/tools/creating_data/download-git-repos.sh b/onap-offline/bash/tools/creating_data/download-git-repos.sh new file mode 100755 index 0000000..9d651d9 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-git-repos.sh @@ -0,0 +1,36 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +lists_dir="$1" + +if [[ -z "$lists_dir" ]]; then + echo "Missing argument for lists_dir" + exit 1 +fi + +outdir="$2" +if [[ -z "$outdir" ]]; then + outdir="./git-repo" +fi + +mkdir -p "$outdir" +cd "$outdir" +# NOTE: will be better to use sh extension? +sh $lists_dir/git_manual_list +sh $lists_dir/git_repos_list + diff --git a/onap-offline/bash/tools/creating_data/download-http-files.sh b/onap-offline/bash/tools/creating_data/download-http-files.sh new file mode 100755 index 0000000..06f4135 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-http-files.sh @@ -0,0 +1,48 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +LIST_FILE="$1" +if [[ -z "$LIST_FILE" ]]; then + echo "Missing list file" + exit 1 +fi + +outdir="$2" +if [[ -z "$outdir" ]]; then + echo "Missing output directory" + exit 1 +fi + +lines=$(cat "$LIST_FILE" | wc -l) +cnt=1 + +# create output dir if not exists +mkdir -p "$outdir" + +while read -r line; do + # www.springframework.org/schema/tool/spring-tool-4.3.xsd + file="${line%%\?*}" + echo "Downloading $cnt / $lines: $file" + fdir=$(dirname "$file") + mkdir -p $outdir/$fdir + # following curl params are ensurring 5 reties and cut-off if connectivity will + # drop below 10b/10s + curl --retry 5 -y 10 -Y 10 --location "$line" -o "$outdir/$file" &>/dev/null + cnt=$((cnt+1)) + +done < "$LIST_FILE" diff --git a/onap-offline/bash/tools/creating_data/download-npm-pkgs.sh b/onap-offline/bash/tools/creating_data/download-npm-pkgs.sh new file mode 100755 index 0000000..9e8847c --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-npm-pkgs.sh @@ -0,0 +1,41 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +LIST_FILE="$1" + +if [[ -z "$LIST_FILE" ]]; then + LIST_FILE="all_npm_list.txt" +fi + +outdir="$2" +if [[ -z "$outdir" ]]; then + echo "Missing arg outdir" + exit 1 +fi + +mkdir -p "$outdir" +cd "$outdir" +lines=$(cat "$LIST_FILE" | wc -l) +cnt=1 +while read -r line; do + echo "== pkg #$cnt of $lines ==" + # yallist@2.1.2 + npm pack $line + cnt=$((cnt+1)) + +done < "$LIST_FILE" diff --git a/onap-offline/bash/tools/creating_data/download-oom.sh b/onap-offline/bash/tools/creating_data/download-oom.sh new file mode 100755 index 0000000..010eeec --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-oom.sh @@ -0,0 +1,37 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +outdir="$1" +if [[ -z "$outdir" ]]; then + echo "Missing output dir" + exit 1 +fi +patch_file="$2" +if [[ -z "$patch_file" ]]; then + echo "Missing patch file" + exit 1 +fi + +cd "$outdir" +git clone https://github.com/onap/oom.git +cd oom +echo "Checkout base commit which will be patched" +git checkout -b patched_beijing bf47d706fc8b94fd1232960e90329a9a532c6a7b + +patch -p1 < "$patch_file" + diff --git a/onap-offline/bash/tools/creating_data/download-pip.sh b/onap-offline/bash/tools/creating_data/download-pip.sh new file mode 100755 index 0000000..dea60b5 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-pip.sh @@ -0,0 +1,45 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +LIST_FILE="$1" +if [[ -z "$LIST_FILE" ]]; then + echo "Missing list file" + exit 1 +fi +LIST_FILE=$(readlink -f "$LIST_FILE") + + +outdir="$2" +if [[ -z "$outdir" ]]; then + echo "Missing output directory" + exit 1 +fi + +lines=$(cat "$LIST_FILE" | wc -l) +cnt=1 + +# create output dir if not exists +mkdir -p "$outdir" + +cd "$outdir" +while read -r line; do + echo "Downloading $cnt / $lines: $line" + pip download $line + cnt=$((cnt+1)) + +done < "$LIST_FILE" diff --git a/onap-offline/bash/tools/creating_data/download-pkg.sh b/onap-offline/bash/tools/creating_data/download-pkg.sh new file mode 100755 index 0000000..03bfe07 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/download-pkg.sh @@ -0,0 +1,42 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +outdir="$1" +if [[ -z "$outdir" ]]; then + echo "Missing output dir" + exit 1 +fi + +# if onap.repo does not exists create it +mkdir -p $outdir +if [ ! -f "$outdir/onap.repo" ]; then + cat > "$outdir/onap.repo" <<EOF +[ONAP] +name=Offline ONAP repository +baseurl=PATH +enabled=1 +gpgcheck=0 +EOF +fi + +# this exact docker version is required by ONAP/beijing +# it should be available in centos docker repo +yumdownloader --resolve --destdir="$outdir" docker-ce-17.03.2.ce libtool-ltdl docker-ce-selinux + +yumdownloader --resolve --destdir="$outdir" dnsmasq icewm firefox tigervnc-server perl* +createrepo "$outdir" diff --git a/onap-offline/bash/tools/creating_data/load-docker-images.sh b/onap-offline/bash/tools/creating_data/load-docker-images.sh new file mode 100755 index 0000000..c6ce152 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/load-docker-images.sh @@ -0,0 +1,74 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +# boilerplate +RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' +if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then + SCRIPT_DIR=$(dirname "${0}") + LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") + . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh +fi + +CLEAN=false + +if [ -z "$NEXUS_HOST" ]; then + echo "Independent run for inserting of additional docker images" + CLEAN=true + mv ~/.docker/config.json ~/.docker/config.json_backup 2>/dev/null + source "$LOCAL_PATH/docker-login.sh" +fi + +IMG_DIR="$1" +if [[ -z "$IMG_DIR" ]]; then + IMG_DIR="$(pwd)/images" +fi + +if [[ ! -d "${IMG_DIR}" ]]; then + echo "No ${IMG_DIR} to load images" + exit 0 +fi + +load_image() { + local image="$1" + echo "[DEBUG] load ${image}" + result=$(docker load -i "${image}") + echo $result + name=$(echo $result | awk '{print $3}') + echo "[DEBUG] pushing $name" + retry docker push "$name" + # delete pushed image from docker + retry docker rmi "$name" +} + +lines=$(ls ${IMG_DIR}/*.tar | wc -l) +line=1 +for image in ${IMG_DIR}/*.tar; do + echo "== pkg #$line of $lines ==" + load_image "$image" + + line=$((line+1)) +done + +if [ "$CLEAN" = true ]; then + # onap is using different credentials for docker login which can be conflicted + # with ours so better to clean this-up + rm ~/.docker/config.json +fi diff --git a/onap-offline/bash/tools/creating_data/make-docker-images-list.sh b/onap-offline/bash/tools/creating_data/make-docker-images-list.sh new file mode 100755 index 0000000..b925d91 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/make-docker-images-list.sh @@ -0,0 +1,23 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +readonly IMG_LIST="/tmp/docker_image_list.txt" + +echo "gathering images" +docker images --digests | grep -v 'IMAGE ID'|awk '{printf("%s:%s %s\n", $1, $2, $3)}' > $IMG_LIST + diff --git a/onap-offline/bash/tools/creating_data/make-git-http-list.sh b/onap-offline/bash/tools/creating_data/make-git-http-list.sh new file mode 100755 index 0000000..a724d6f --- /dev/null +++ b/onap-offline/bash/tools/creating_data/make-git-http-list.sh @@ -0,0 +1,74 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +if [[ -z "$LISTS_DIR" ]]; then + LISTS_DIR=. + echo "Using default output directory ." +fi + +OOM_PATH="$1" + +if [[ -z "$OOM_PATH" ]]; then + echo "Missing oom path" + exit 1 +fi + + +GOUTPUT="$LISTS_DIR/git_repos_list" +FOUTPUT="$LISTS_DIR/fetch_list.txt" + +trim_last() { + echo "${@:1:$#-1}"; +} + +TMP='/tmp/git_tmp_list' +:> $TMP + +:> $FOUTPUT + +echo "Gathering git repos and list possible html data" + +while read -r chart; do + out="$(helm template $(dirname "$chart") 2>/dev/null)" + gitcmds=$(echo "$out" | grep 'git clone') + + if [[ -n "$gitcmds" ]] ; then + while read gitcmd; do + gitcmd=$(trim_last $gitcmd) + repo_path=$(echo "$gitcmd" | sed 's#.*http://\(.*\)$#\1#') + full="$gitcmd --bare $repo_path" + echo "Cmd: $full" + echo "$full" >> $TMP + done <<< "$gitcmds" + fi + + fetchcmds=$(echo "$out" | grep 'wget \|curl ' | grep -v 'HEALTH_CHECK_ENDPOINT\|PUT\|POST' ) + if [[ -n "$fetchcmds" ]] ; then + while read fetchcmd; do + echo "Fetch: $fetchcmd" + echo "=== From: $chart" >> $FOUTPUT + echo "$fetchcmd" >> $FOUTPUT + done <<< "$fetchcmds" + fi + + +done <<< "$(find $OOM_PATH -name Chart.yaml)" + +sort $TMP | uniq > $GOUTPUT + + diff --git a/onap-offline/bash/tools/creating_data/make-npm-list.sh b/onap-offline/bash/tools/creating_data/make-npm-list.sh new file mode 100755 index 0000000..5901d54 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/make-npm-list.sh @@ -0,0 +1,35 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +FIND_PATH="/var/lib/docker/aufs/mnt/" + +LIST_FILE="/tmp/npm.list" + +: > $LIST_FILE + +echo "gathering npm packages from $FIND_PATH" +while read -r line; do + # yallist/2.1.2/package/package.json + name=$(echo $line | cut -d '/' -f1 ) + ver=$(echo $line | cut -d '/' -f2 ) +# tag=$(echo $image|awk '{print $2}') +# save_image "$name:$tag" + echo "$name@$ver" >> $LIST_FILE + +done <<< "$(find $FIND_PATH -path "*/.npm/*/package.json" | sed 's#^.*\.npm/\(.*\)$#\1#' | sort | uniq)" + diff --git a/onap-offline/bash/tools/creating_data/remote-list-gathering.sh b/onap-offline/bash/tools/creating_data/remote-list-gathering.sh new file mode 100755 index 0000000..73fb164 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/remote-list-gathering.sh @@ -0,0 +1,53 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +if [[ -z "$ONAP_SERVERS" ]]; then +# ONAP_SERVERS="oom-beijing-postRC2-master oom-beijing-postRC2-compute1 oom-beijing-postRC2-compute2" + echo "Missing environment ONAP_SERVERS" + exit 1 +fi +if [[ -z "$LISTS_DIR" ]]; then + LISTS_DIR=. + echo "Using default output directory ." +fi + +echo "Using onap servers: $ONAP_SERVERS" + +cd $(dirname $0) + +for server in $ONAP_SERVERS; do + + echo "=================================================" + echo "Gathering docker images list from server: $server" + echo "=================================================" + scp ./make-docker-images-list.sh $server:/tmp/ + ssh $server '/tmp/make-docker-images-list.sh;rm /tmp/make-docker-images-list.sh' + scp $server:/tmp/docker_image_list.txt $LISTS_DIR/docker_image_list-${server}.txt + + echo "=================================================" + echo "Gathering NPM packages list from server: $server" + echo "=================================================" + scp ./make-npm-list.sh $server:/tmp/ + ssh $server '/tmp/make-npm-list.sh; rm /tmp/make-npm-list.sh' + scp $server:/tmp/npm.list $LISTS_DIR/npm_list-${server}.txt +done + +cat "$LISTS_DIR"/docker_image_list-*.txt | grep -v sonatype/nexus3 | grep -v own_nginx | sort | uniq > "$LISTS_DIR/docker_image_list.txt" +cat "$LISTS_DIR"/npm_list-*.txt | grep -v '^@$' | sort | uniq > "$LISTS_DIR/npm_list.txt" + diff --git a/onap-offline/bash/tools/creating_data/save-docker-images.sh b/onap-offline/bash/tools/creating_data/save-docker-images.sh new file mode 100755 index 0000000..f4a5d3c --- /dev/null +++ b/onap-offline/bash/tools/creating_data/save-docker-images.sh @@ -0,0 +1,67 @@ +#! /usr/bin/env bash + +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + + +# boilerplate +RELATIVE_PATH=../ # relative path from this script to 'common-functions.sh' +if [ "$IS_COMMON_FUNCTIONS_SOURCED" != YES ] ; then + SCRIPT_DIR=$(dirname "${0}") + LOCAL_PATH=$(readlink -f "$SCRIPT_DIR") + . "${LOCAL_PATH}"/"${RELATIVE_PATH}"/common-functions.sh +fi + +IMG_DIR="$1" + +if [[ -z "$IMG_DIR" ]]; then + IMG_DIR="./images" +fi + +echo "Creating ${IMG_DIR}" +if [[ ! -d "${IMG_DIR}" ]]; then + mkdir -p "${IMG_DIR}" +fi + +save_image() { + local name_tag=$1 + echo "$name_tag" + local img_name=$(echo "${name_tag}" | tr /: __) + local img_path="${IMG_DIR}/${img_name}.tar" + + if [[ ! -f "${img_path}" ]] ; then + echo "[DEBUG] save ${name_tag} to ${img_path}" + echo "${name_tag}" >> $IMG_DIR/_image_list.txt + retry docker -l error save -o "${img_path}" ${name_tag} + else + echo "[DEBUG] ${name_tag} already saved" + fi +} + +echo "Save all images" +line=1 +lines=$(docker images|grep -v 'IMAGE ID'|wc -l) +while read -r image; do + echo "== pkg #$line of $lines ==" + + name=$(echo $image|awk '{print $1}') + tag=$(echo $image|awk '{print $2}') + save_image "$name:$tag" + line=$((line+1)) + +done <<< "$(docker images|grep -v 'IMAGE ID'|awk '{printf("%s %s\n", $1, $2)}'|column -t)" diff --git a/onap-offline/bash/tools/creating_data/upload-maven-files.sh b/onap-offline/bash/tools/creating_data/upload-maven-files.sh new file mode 100755 index 0000000..07d6e7a --- /dev/null +++ b/onap-offline/bash/tools/creating_data/upload-maven-files.sh @@ -0,0 +1,43 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +DATA_DIR="$1" +if [[ -z "$DATA_DIR" ]]; then + # needs for example: /root/onap-offline-installer/http + echo "Mising arg DATA_DIR" + echo "Usage: $0 <path to http dir> <name of server> [<next server>...]" + exit 1 +fi + +shift +cd "$DATA_DIR" + +for server in $*; do + echo "Uploading to server: $server" + + lines=$(find $server/ -type f | wc -l) + count=1 + while read -r url; do + echo "== pkg #$count of $lines ==" + count=$((count + 1)) + path="$url" + echo "Uploading file: $url" + curl -u admin:admin123 --upload-file $path http://$url + + done <<< "$(find $server/ -type f)" +done diff --git a/onap-offline/bash/tools/creating_data/upload-npm-pkgs.sh b/onap-offline/bash/tools/creating_data/upload-npm-pkgs.sh new file mode 100755 index 0000000..9a7ed55 --- /dev/null +++ b/onap-offline/bash/tools/creating_data/upload-npm-pkgs.sh @@ -0,0 +1,48 @@ +# COPYRIGHT NOTICE STARTS HERE +# +# Copyright 2018 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# COPYRIGHT NOTICE ENDS HERE + +LIST_FILE="$1" +if [[ -z "$LIST_FILE" ]]; then + echo "Mising arg LIST_FILE" + exit 1 +fi + +DATA_DIR="$2" +if [[ -z "$DATA_DIR" ]]; then + echo "Mising arg DATA_DIR" + exit 1 +fi + +NEXUS_HOST="$3" +if [[ -z "$NEXUS_HOST" ]]; then + echo "Mising arg NEXUS_HOST" + exit 1 +fi + +npm config set registry $NEXUS_HOST/repository/npm-private/ +# npm adduser moved to top + +cd "$DATA_DIR" +lines=$(ls *.tgz | wc -l) +cnt=1 +for line in *.tgz; do + echo "== pkg #$cnt of $lines ==" + # yallist@2.1.2 + npm publish --access public "${line}" + cnt=$((cnt+1)) +done |