From 4d147ebcb1e732721b9de11e894063936c4e3b6b Mon Sep 17 00:00:00 2001 From: Donald Hunter Date: Fri, 8 Mar 2019 14:56:55 +0000 Subject: Move PNDA mirror and bootstrap from deployments Change-Id: I1c23c9a37f27f0fae46ae1547896d1462f67a15c Issue-ID: DCAEGEN2-1322 Signed-off-by: Donald Hunter --- mvn-phase-script.sh | 104 ++++++++++++ pnda-bootstrap-container/Dockerfile | 34 ++++ pnda-bootstrap-container/README.md | 8 + pnda-bootstrap-container/pom.xml | 256 +++++++++++++++++++++++++++++ pnda-mirror-container/Dockerfile | 47 ++++++ pnda-mirror-container/README.md | 23 +++ pnda-mirror-container/pnda-5.0-maint.patch | 25 +++ pnda-mirror-container/pom.xml | 256 +++++++++++++++++++++++++++++ pom.xml | 2 + 9 files changed, 755 insertions(+) create mode 100755 mvn-phase-script.sh create mode 100644 pnda-bootstrap-container/Dockerfile create mode 100644 pnda-bootstrap-container/README.md create mode 100644 pnda-bootstrap-container/pom.xml create mode 100644 pnda-mirror-container/Dockerfile create mode 100644 pnda-mirror-container/README.md create mode 100644 pnda-mirror-container/pnda-5.0-maint.patch create mode 100644 pnda-mirror-container/pom.xml diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh new file mode 100755 index 0000000..590ed4c --- /dev/null +++ b/mvn-phase-script.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# ================================================================================ +# Copyright (c) 2017-2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# + + +set -ex + +echo "running script: [$0] for module [$1] at stage [$2]" + +MVN_PROJECT_MODULEID="$1" +MVN_PHASE="$2" +PROJECT_ROOT=$(dirname $0) + + +echo "MVN_RELEASE_TAG is set to [$MVN_RELEASE_TAG]" +RELEASE_TAG=${MVN_RELEASE_TAG:-R4} +if [ "$RELEASE_TAG" != "R1" ]; then + RELEASE_TAGGED_DIR="${RELEASE_TAG}/" +else + RELEASE_TAGGED_DIR="" +fi +if ! wget -O ${PROJECT_ROOT}/mvn-phase-lib.sh \ + "$MVN_RAWREPO_BASEURL_DOWNLOAD"/org.onap.dcaegen2.utils/${RELEASE_TAGGED_DIR}scripts/mvn-phase-lib.sh; then + echo "Fail to download mvn-phase-lib.sh" + exit 1 +fi +source "${PROJECT_ROOT}"/mvn-phase-lib.sh + + +TIMESTAMP=$(date +%C%y%m%dT%H%M%S) +export BUILD_NUMBER="${TIMESTAMP}" +shift 2 + +case $MVN_PHASE in +clean) + echo "==> clean phase script" + clean_templated_files + clean_tox_files + rm -rf ./venv-* ./*.wgn ./site ./coverage.xml ./xunit-results.xml + ;; +generate-sources) + echo "==> generate-sources phase script" + expand_templates + ;; +compile) + echo "==> compile phase script" + ;; +test) + echo "==> test phase script" + ;; +package) + echo "==> package phase script" + ;; +install) + echo "==> install phase script" + case $MVN_PROJECT_MODULEID in + bootstrap) + upload_files_of_extension sh + ;; + esac + ;; +deploy) + echo "==> deploy phase script" + + case $MVN_PROJECT_MODULEID in + bootstrap) + # build docker image from Docker file (under module dir) and push to registry + upload_files_of_extension sh + build_and_push_docker + ;; + k8s-bootstrap-container|tca-cdap-container|cm-container|redis-cluster-container|healthcheck-container|pnda-mirror-container|pnda-bootstrap-container|tls-init-container|consul-loader-container) + build_and_push_docker + ;; + scripts|cloud_init|heat) + # upload all sh file under the root of module + upload_files_of_extension_recursively sh $MVN_PROJECT_MODULEID + upload_files_of_extension_recursively py $MVN_PROJECT_MODULEID + upload_files_of_extension_recursively yaml $MVN_PROJECT_MODULEID + ;; + *) + echo "====> unknown mvn project module" + ;; + esac + ;; +*) + echo "==> unprocessed phase" + ;; +esac + diff --git a/pnda-bootstrap-container/Dockerfile b/pnda-bootstrap-container/Dockerfile new file mode 100644 index 0000000..4fefa8a --- /dev/null +++ b/pnda-bootstrap-container/Dockerfile @@ -0,0 +1,34 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 Cisco Systems. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +FROM python:2.7.15 as build + +ARG PNDACLITAG=release/5.0 +ARG PLATFORMSALTTAG=release/5.0 + +RUN git clone -b $PNDACLITAG https://github.com/pndaproject/pnda-cli.git +RUN git clone -b $PLATFORMSALTTAG https://github.com/pndaproject/platform-salt.git + +RUN pip2 install --no-cache-dir -r pnda-cli/cli/requirements.txt --install-option="--prefix=/install" + +FROM python:2.7.15-alpine3.8 +COPY --from=build /install /usr/local +COPY --from=build /pnda-cli /pnda-cli +COPY --from=build /platform-salt /platform-salt + +RUN apk add --no-cache curl jq openssl openssh diff --git a/pnda-bootstrap-container/README.md b/pnda-bootstrap-container/README.md new file mode 100644 index 0000000..43e089e --- /dev/null +++ b/pnda-bootstrap-container/README.md @@ -0,0 +1,8 @@ +# PNDA Boostrap container +## Purpose +The artifacts in this directory build a Docker image including the PNDA CLI. +The CLI allows the bootstrap of a PNDA container. + +## Running the Container +The container is intended to be launched via a Helm chart as part +of the ONAP deployment process, guided by OOM. diff --git a/pnda-bootstrap-container/pom.xml b/pnda-bootstrap-container/pom.xml new file mode 100644 index 0000000..1b173ed --- /dev/null +++ b/pnda-bootstrap-container/pom.xml @@ -0,0 +1,256 @@ + + + + 4.0.0 + + org.onap.dcaegen2.analytics.pnda + pnda-utils + 1.0.0-SNAPSHOT + + org.onap.dcaegen2.deployments + pnda-bootstrap-container + dcaegen2-deployments-pnda-bootstrap-container + 5.0.0 + http://maven.apache.org + + UTF-8 + true + . + py + Python + **/*.py + + https://nexus.onap.org + + https://nexus.onap.org/content/sites/raw + https://nexus.onap.org/service/local/repositories/raw/content + ecomp-raw + + nexus3.onap.org:10003 + nexus3.onap.org:10002 + nexus3.onap.org:10003 + nexus3.onap.org:10002 + + https://nexus3.onap.org/repository/PyPi + onap-pypi + + + ${project.artifactId}-${project.version} + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + + true + true + + + + org.apache.maven.plugins + maven-deploy-plugin + + 2.8 + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.6 + + true + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + true + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + default-jar + + + + + + + org.apache.maven.plugins + maven-install-plugin + 2.4 + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + ../mvn-phase-script.sh + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + ${onap.nexus.rawrepo.baseurl.upload} + ${onap.nexus.rawrepo.baseurl.download} + ${onap.nexus.rawrepo.serverid} + ${onap.nexus.dockerregistry.snapshot} + ${onap.nexus.dockerregistry.release} + ${onap.nexus.dockerregistry.snapshot.serverid} + ${onap.nexus.dockerregistry.release.serverid} + ${onap.nexus.pypiserver.baseurl} + ${onap.nexus.pypiserver.serverid} + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + clean phase script + clean + + exec + + + + ${project.artifactId} + clean + + + + + generate-sources script + generate-sources + + exec + + + + ${project.artifactId} + generate-sources + + + + + compile script + compile + + exec + + + + ${project.artifactId} + compile + + + + + package script + package + + exec + + + + ${project.artifactId} + package + + + + + test script + test + + exec + + + + ${project.artifactId} + test + + + + + install script + install + + exec + + + + ${project.artifactId} + install + + + + + deploy script + deploy + + exec + + + + ${project.artifactId} + deploy + + + + + + + + diff --git a/pnda-mirror-container/Dockerfile b/pnda-mirror-container/Dockerfile new file mode 100644 index 0000000..ea7e100 --- /dev/null +++ b/pnda-mirror-container/Dockerfile @@ -0,0 +1,47 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2018 Cisco Systems. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +FROM centos:7.5.1804 as build + +ARG PNDARELEASE=release/5.0 +# Can be HDP or CDH +ARG HADOOPDIST=HDP + +RUN yum clean all && rm -rf /var/cache/yum/* && yum install gettext git -y + +RUN git clone -b $PNDARELEASE https://github.com/pndaproject/pnda.git + +COPY pnda-5.0-maint.patch / +WORKDIR /pnda +RUN git apply /pnda-5.0-maint.patch + +WORKDIR /pnda/mirror +# Add the -r flag to mirror rpm packages +RUN ./create_mirror.sh -d $HADOOPDIST -r + +WORKDIR /pnda/build +RUN ./install-build-tools.sh + +RUN yum install bzip2 make which -y +RUN source ./set-pnda-env.sh \ + && PARALLEL="--jobs 1" ./build-pnda.sh RELEASE $PNDARELEASE $HADOOPDIST + +FROM nginx:alpine + +COPY --from=build /pnda/mirror/mirror-dist /usr/share/nginx/html/ +COPY --from=build /pnda/build/pnda-dist /usr/share/nginx/html/ diff --git a/pnda-mirror-container/README.md b/pnda-mirror-container/README.md new file mode 100644 index 0000000..6312439 --- /dev/null +++ b/pnda-mirror-container/README.md @@ -0,0 +1,23 @@ +# PNDA Mirror +## Purpose +The artifacts in this directory build a Docker image based public PNDA mirror +creation scripts. The container have all the needed offline resources to +deploy a PNDA platform. + +## Running the Container +The container is intended to be launched via a Helm chart as part +of the ONAP deployment process, guided by OOM. It can be run directly +into a native Docker environment, using: +``` +docker run --name pnda-mirror -d --restart unless-stopped \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -p :8080 \ + --tmpfs /run \ + --tmpfs /run/lock \ + --security-opt seccomp:unconfined + --cap-add SYS_ADMIN \ + +``` + +We also expect that in a Kubernetes environment the external port mapping would not be +needed. diff --git a/pnda-mirror-container/pnda-5.0-maint.patch b/pnda-mirror-container/pnda-5.0-maint.patch new file mode 100644 index 0000000..437b0db --- /dev/null +++ b/pnda-mirror-container/pnda-5.0-maint.patch @@ -0,0 +1,25 @@ +diff --git a/build/upstream-builds/build-gobblin.sh b/build/upstream-builds/build-gobblin.sh +index d191524..34ee0a0 100755 +--- a/build/upstream-builds/build-gobblin.sh ++++ b/build/upstream-builds/build-gobblin.sh +@@ -17,7 +17,7 @@ MODE=${1} + ARG=${2} + HADOOP_DISTRIBUTION=${3} + +-EXCLUDES="-x test" ++EXCLUDES="-x test -x findbugsMain" + set -e + set -x + +diff --git a/mirror/dependencies/pnda-static-file-dependencies.txt b/mirror/dependencies/pnda-static-file-dependencies.txt +index 8f04a87..133e08b 100644 +--- a/mirror/dependencies/pnda-static-file-dependencies.txt ++++ b/mirror/dependencies/pnda-static-file-dependencies.txt +@@ -21,5 +21,5 @@ http://central.maven.org/maven2/org/kitesdk/kite-tools/1.0.0/kite-tools-1.0.0-bi + http://central.maven.org/maven2/org/kitesdk/kite-tools/1.0.0/kite-tools-1.0.0-binary.jar.sha1 + https://releases.hashicorp.com/consul/1.0.3/consul_1.0.3_linux_amd64.zip + https://releases.hashicorp.com/consul/1.0.3/consul_1.0.3_SHA256SUMS +-http://www.apache.org/dist/knox/1.1.0/knox-1.1.0.zip +-http://www.apache.org/dist/knox/1.1.0/knox-1.1.0.zip.sha1 ++http://archive.apache.org/dist/knox/1.1.0/knox-1.1.0.zip ++http://archive.apache.org/dist/knox/1.1.0/knox-1.1.0.zip.sha1 diff --git a/pnda-mirror-container/pom.xml b/pnda-mirror-container/pom.xml new file mode 100644 index 0000000..09a42f0 --- /dev/null +++ b/pnda-mirror-container/pom.xml @@ -0,0 +1,256 @@ + + + + 4.0.0 + + org.onap.dcaegen2.analytics.pnda + pnda-utils + 1.0.0-SNAPSHOT + + org.onap.dcaegen2.deployments + pnda-mirror-container + dcaegen2-deployments-pnda-mirror-container + 5.0.0 + http://maven.apache.org + + UTF-8 + true + . + py + Python + **/*.py + + https://nexus.onap.org + + https://nexus.onap.org/content/sites/raw + https://nexus.onap.org/service/local/repositories/raw/content + ecomp-raw + + nexus3.onap.org:10003 + nexus3.onap.org:10002 + nexus3.onap.org:10003 + nexus3.onap.org:10002 + + https://nexus3.onap.org/repository/PyPi + onap-pypi + + + ${project.artifactId}-${project.version} + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + + true + true + + + + org.apache.maven.plugins + maven-deploy-plugin + + 2.8 + + true + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.6 + + true + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + true + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + default-jar + + + + + + + org.apache.maven.plugins + maven-install-plugin + 2.4 + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + ../mvn-phase-script.sh + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + ${onap.nexus.rawrepo.baseurl.upload} + ${onap.nexus.rawrepo.baseurl.download} + ${onap.nexus.rawrepo.serverid} + ${onap.nexus.dockerregistry.snapshot} + ${onap.nexus.dockerregistry.release} + ${onap.nexus.dockerregistry.snapshot.serverid} + ${onap.nexus.dockerregistry.release.serverid} + ${onap.nexus.pypiserver.baseurl} + ${onap.nexus.pypiserver.serverid} + + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + clean phase script + clean + + exec + + + + ${project.artifactId} + clean + + + + + generate-sources script + generate-sources + + exec + + + + ${project.artifactId} + generate-sources + + + + + compile script + compile + + exec + + + + ${project.artifactId} + compile + + + + + package script + package + + exec + + + + ${project.artifactId} + package + + + + + test script + test + + exec + + + + ${project.artifactId} + test + + + + + install script + install + + exec + + + + ${project.artifactId} + install + + + + + deploy script + deploy + + exec + + + + ${project.artifactId} + deploy + + + + + + + + diff --git a/pom.xml b/pom.xml index 6b193d5..6182853 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,8 @@ limitations under the License. pnda-ztt-app + pnda-mirror-container + pnda-bootstrap-container -- cgit 1.2.3-korg