diff options
| -rw-r--r-- | cm-container/Dockerfile-template | 21 | ||||
| -rwxr-xr-x | cm-container/findPlugins.sh | 74 | ||||
| -rw-r--r-- | cm-container/pom.xml | 2 | ||||
| -rwxr-xr-x | cm-container/scripts/get-plugins.sh | 60 | ||||
| -rwxr-xr-x | cm-container/scripts/get-type-files.sh | 77 | ||||
| -rwxr-xr-x | cm-container/scripts/load-plugins.sh | 103 | ||||
| -rw-r--r-- | cm-container/scripts/readiness-check.sh | 8 | ||||
| -rwxr-xr-x | cm-container/scripts/set-resolver-rules.sh | 39 | ||||
| -rw-r--r-- | cm-container/scripts/update_resolver.py | 56 |
9 files changed, 249 insertions, 191 deletions
diff --git a/cm-container/Dockerfile-template b/cm-container/Dockerfile-template index b392e92..5e6d846 100644 --- a/cm-container/Dockerfile-template +++ b/cm-container/Dockerfile-template @@ -1,7 +1,5 @@ # ============LICENSE_START======================================================= -# org.onap.dcae -# ================================================================================ -# Copyright (c) 2018-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,27 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============LICENSE_END========================================================= -# -# ECOMP is a trademark and service mark of AT&T Intellectual Property. FROM cloudifyplatform/community:19.01.24 -MAINTAINER maintainer -ENV TYPE_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_plugins_releases }} -ENV CCSDK_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }} +ENV PLUGIN_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_plugins_releases }} -# Store type files locally +# Store plugin files locally RUN mkdir scripts COPY scripts/* scripts/ -# Load our type files and the Cloudify 3.4 type files +# Load our plugin files # Setup rc.local to set up k8s credentials for CM -RUN scripts/get-type-files.sh ${TYPE_REPO} ${CCSDK_REPO}\ - && mkdir /opt/manager/resources/spec/cloudify/3.4\ - && curl -Ss -L -f https://getcloudify.org/spec/cloudify/3.4/types.yaml > /opt/manager/resources/spec/cloudify/3.4/types.yaml\ - && chown -R cfyuser:cfyuser /opt/manager/resources/spec/cloudify/3.4\ +RUN scripts/get-plugins.sh ${PLUGIN_REPO} \ && chmod +x scripts/*.sh\ && /scripts/configure-tls.sh\ && echo "/scripts/setup-secret.sh" >> /etc/rc.d/rc.local\ - && echo "/scripts/set-resolver-rules.sh" >> /etc/rc.d/rc.local\ + && echo "/scripts/load-plugins.sh > /load-plugins.log 2>&1" >> /etc/rc.d/rc.local\ && chmod +x /etc/rc.d/rc.local # Create mount point for CM config file RUN mkdir -p /opt/onap && chown cfyuser:cfyuser /opt/onap diff --git a/cm-container/findPlugins.sh b/cm-container/findPlugins.sh new file mode 100755 index 0000000..10d49a7 --- /dev/null +++ b/cm-container/findPlugins.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# ============LICENSE_START======================================================= +# Copyright (c)2020 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +# Use the Nexus API to get a list of all the plugins/typefiles currently in Nexus +# under a specified root directory, passed as the first argument to the script +# We assume the repo structure that the build process uses: +# PLUGIN_ROOT/<plugin_name>/<version>/<wagon_name>.wgn +# PLUGIN_ROOT/<plugin_name>/<version>/<type_file_name>.yaml + +# This code could be used as the basis for an alternative to the existing +# 'get-plugins.sh' script. Instead of pulling a hard-coded list of plugins and +# type files from Nexus, it would pull all of the plugins and type files, in all +# available versions, from the Nexus repo. + +# At the very least, it is a useful tool for finding out what plugins and +# type files have been loaded into the Nexus repo and are therefore available +# to be included in the hard-coded list. + +shopt -s expand_aliases + +alias cu='curl -Ss -H "Accept: application/json" -L -f' +PLUGIN_ROOT=${1:-"https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R7/"} + +function getPlugins() { + local root=$1 + # Get URLs for all of the plugins under the plugin root directory + local PLUGINS=$(cu $root | jq .data[].resourceURI | sed -e 's/"//g') + + # Go into each plugin directory + for p in $PLUGINS + do + # Get the available versions of the plugin + local VERSIONS=$(cu $p | jq .data[].resourceURI | sed -e 's/"//g') + + # Get wagon and type file for each version + for v in $VERSIONS + do + local RESOURCES=$(cu $v | jq .data[].relativePath) + # RESOURCES will have a list of everything in the version directory, including many timestamped + # wagons and type files, and some zip files as well. For each version, there should be a single + # non-time-stamped .wgn and a single non-timestamped .yaml file. We try to pull these from the + # from the list using grep, and then we reformat the results to remove quote marks and to remove + # the first two levels of the relative path. + # Just in case we're wrong about how many non-timestamped .wgn and .yaml files are in each directory, + # we treat the result of the grep as an array and we take the first element only. + local w=($((for r in $RESOURCES; do echo $r; done) | grep ".wgn\"" | tr -d '"' | sed -e 's#^/[^/]*/[^/]*##')) + local t=($((for r in $RESOURCES; do echo $r; done) | grep ".yaml\"" | tr -d '"' | sed -e 's#^/[^/]*/[^/]*##')) + echo "${w[0]}|${t[0]}" + # We could potentially fetch the plugin wagon file and the type file here. Probably wouldn't need + # to have this code as a function in that case. Also, probably should the .resourceURI (the full URL + # of each file) rather than .relativePath, and not strip off any part of the path (the 'sed' operation + # would not be needed.) + done + done +} + +for p in $(getPlugins $PLUGIN_ROOT | sort) +do + echo $p +done diff --git a/cm-container/pom.xml b/cm-container/pom.xml index 207e979..4da4d8c 100644 --- a/cm-container/pom.xml +++ b/cm-container/pom.xml @@ -27,7 +27,7 @@ limitations under the License. <groupId>org.onap.dcaegen2.deployments</groupId> <artifactId>cm-container</artifactId> <name>dcaegen2-deployments-cm-container</name> - <version>2.1.0</version> + <version>3.0.0</version> <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> diff --git a/cm-container/scripts/get-plugins.sh b/cm-container/scripts/get-plugins.sh new file mode 100755 index 0000000..7c2f353 --- /dev/null +++ b/cm-container/scripts/get-plugins.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# ============LICENSE_START======================================================= +# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= + +# Pull plugin wagon files and type files from repo +# $1 is the DCAE repo URL - script assumes all files come from the +# same repo, but not necessarily same paths +# +set -x -e + +# Location in CM container where plugins/type files will be stored +# At deployment, CM script will look here to find plugins to upload +DEST=${DEST:-/opt/plugins} + +# Each line has a plugin wagon/type file pair, in the form +# /path/to/plugin/wagon|/path/to/type/file +PLUGINS=\ +"\ +/dcaepolicyplugin/2.4.0/dcaepolicyplugin-2.4.0-py27-none-linux_x86_64.wgn|/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \ +/relationshipplugin/1.1.0/relationshipplugin-1.1.0-py27-none-linux_x86_64.wgn|/relationshipplugin/1.1.0/relationshipplugin_types.yaml \ +/k8splugin/2.0.0/k8splugin-2.0.0-py27-none-linux_x86_64.wgn|/k8splugin/2.0.0/k8splugin_types.yaml \ +/k8splugin/3.0.0/k8splugin-3.0.0-py27-none-linux_x86_64.wgn|/k8splugin/3.0.0/k8splugin_types.yaml \ +/clamppolicyplugin/1.1.0/clamppolicyplugin-1.1.0-py27-none-linux_x86_64.wgn|/clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \ +/dmaap/1.5.0/dmaap-1.5.0-py27-none-linux_x86_64.wgn|/dmaap/1.5.0/dmaap_types.yaml \ +/helm/4.2.0/helm-4.2.0-py27-none-linux_x86_64.wgn|/helm/4.2.0/helm_types.yaml \ +/pgaas/1.3.0/pgaas-1.3.0-py27-none-linux_x86_64.wgn|/pgaas/1.3.0/pgaas_types.yaml \ +/sshkeyshare/1.2.0/sshkeyshare-1.2.0-py27-none-linux_x86_64.wgn|/sshkeyshare/1.2.0/sshkeyshare_types.yaml +" + +mkdir -p ${DEST} + +for p in ${PLUGINS} +do + w=$(echo $p | cut -d '|' -f1) + t=$(echo $p | cut -d '|' -f2) + + # Put each wagon/type file pair into its own subdirectory + # This prevents name collisions which can happen because + # type files don't embed a version. + subdir=$(mktemp -d -t plugin-XXXXXXX --tmpdir=${DEST}) + + curl -Ss -L -f $1/$t >> ${subdir}/$(basename $t) + curl -Ss -L -f $1/$w >> ${subdir}/$(basename $w) + +done + +chown -R cfyuser:cfyuser ${DEST} diff --git a/cm-container/scripts/get-type-files.sh b/cm-container/scripts/get-type-files.sh deleted file mode 100755 index 3b25bbb..0000000 --- a/cm-container/scripts/get-type-files.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -# ============LICENSE_START======================================================= -# org.onap.dcae -# ================================================================================ -# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= - -# Pull type files from repos -# Set up the CM import resolver -# $1 is the DCAE repo URL -# $2 is the CCSDK repo URL -# -set -x -DEST=/opt/manager/resources/onapspec -EXTRA_RULES=/opt/manager/extra-resolver-rules - -DCAETYPEFILES=\ -"\ -/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \ -/relationshipplugin/1.1.0/relationshipplugin_types.yaml \ -/k8splugin/1.7.2/k8splugin_types.yaml \ -/k8splugin/2.0.0/k8splugin_types.yaml \ -clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \ - -" - -CCSDKTYPEFILES=\ -"\ -/type_files/pgaas/1.1.0/pgaas_types.yaml \ -/type_files/sshkeyshare/sshkey_types.yaml \ -/type_files/helm/4.0.2/helm-type.yaml \ -/type_files/dmaap/dmaap.yaml \ -" - -mkdir ${DEST} - -for typefile in ${DCAETYPEFILES} -do - mkdir -p ${DEST}/$(dirname ${typefile}) - curl -Ss -L -f $1/${typefile} >> ${DEST}/${typefile} -done - -for typefile in ${CCSDKTYPEFILES} -do - mkdir -p ${DEST}/$(dirname ${typefile}) - curl -Ss -L -f $2/${typefile} >> ${DEST}/${typefile} -done - -chown cfyuser:cfyuser ${DEST} - -# Add our local type file store to CM import resolver configuration -TYPE_RULE0="{\"$1\": \"file://${DEST}\"}" -TYPE_RULE1="{\"$2\": \"file://${DEST}\"}" -# This sed re is 'brittle' but we can be sure the config.yaml file -# won't change as long as we do not change the source Docker image for CM -sed -i -e "s# rules:# rules:\n - ${TYPE_RULE0}#" /etc/cloudify/config.yaml -sed -i -e "s# rules:# rules:\n - ${TYPE_RULE1}#" /etc/cloudify/config.yaml - -chown cfyuser:cfyuser /etc/cloudify/config.yaml - -# Changing /etc/cloudify/config.yaml is no longer sufficient -# Need to provide the additional rules in a file that can be -# used at deployment time to update the resolver rules -echo "- ${TYPE_RULE0}" > ${EXTRA_RULES} -echo "- ${TYPE_RULE1}" >> ${EXTRA_RULES} diff --git a/cm-container/scripts/load-plugins.sh b/cm-container/scripts/load-plugins.sh new file mode 100755 index 0000000..f4d1f66 --- /dev/null +++ b/cm-container/scripts/load-plugins.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# ============LICENSE_START======================================================= +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# Runs at deployment time to load the plugins/type files +# that were stored into the container file system at build time + +PLUGINS_LOADED=/opt/manager/plugins-loaded +PLUGIN_DIR=/opt/plugins + +# Set defaults for CM address, protocol, and port +CMADDR=${CMADDR:-dcae-cloudify-manager} +CMPROTO=${CMPROTO:-https} +CMPORT=${CMPORT:-443} + +# Password is currently fixed at the default +# Eventually the password will be passed in as an environment variable +CMPASS=${CMPASS:-admin} + +# Set up additional parameters for using HTTPS +CACERT="/opt/onap/certs/cacert.pem" +CURLTLS="" +if [ $CMPROTO = "https" ] +then + CURLTLS="--cacert $CACERT" +fi + +### FUNCTION DEFINITIONS ### + +# cm_hasany: Query Cloudify Manager and return 0 (true) if there are any entities matching the query +# Used to see if something is already present on CM +# $1 -- query fragment, for instance "plugins?archive_name=xyz.wgn" to get +# the number of plugins that came from the archive file "xyz.wgn" +function cm_hasany { + # We use _include=id to limit the amount of data the CM sends back + # We rely on the "metadata.pagination.total" field in the response + # for the total number of matching entities + COUNT=$(curl -Ss -H "Tenant: default_tenant" --user admin:${CMPASS} ${CURLTLS} "${CMPROTO}://${CMADDR}:${CMPORT}/api/v3.1/$1&_include=id" \ + | /bin/jq .metadata.pagination.total) + if (( $COUNT > 0 )) + then + return 0 + else + return 1 + fi +} + +# Install plugin if it's not already installed +# $1 -- path to wagon file for plugin +# $2 -- path to type file for plugin +function install_plugin { + ARCHIVE=$(basename $1) + # See if it's already installed + if cm_hasany "plugins?archive_name=$ARCHIVE" + then + echo plugin $1 already installed on ${CMADDR} + else + cfy plugin upload -y $2 $1 + fi +} + +### END FUNCTION DEFINTIONS ### + +set -ex + + +# Wait for Cloudify Manager to come up +while ! /scripts/cloudify-ready.sh +do + echo "Waiting for CM to come up" + sleep 15 +done + +if [[ ! -f ${PLUGINS_LOADED} ]] +then + + # Each subdirectory of ${PLUGIN_DIR} contains a wagon (.wgn) and type file (.yaml) + for p in ${PLUGIN_DIR}/* + do + # Expecting exactly 1 .wgn and 1 .yaml + # But just in case, taking only the first of each + # (If either is missing, will fail on install_plugin) + wagons=($p/*.wgn) + types=($p/*.yaml) + install_plugin ${wagons[0]} ${types[0]} + done + + touch ${PLUGINS_LOADED} +else + echo "Plugins already loaded" +fi
\ No newline at end of file diff --git a/cm-container/scripts/readiness-check.sh b/cm-container/scripts/readiness-check.sh index a7e92ab..d706c0d 100644 --- a/cm-container/scripts/readiness-check.sh +++ b/cm-container/scripts/readiness-check.sh @@ -2,7 +2,7 @@ # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,12 +24,14 @@ # /opt/manager/extra-resolver-rules-loaded. # -- All Cloudify Manager services must be running, as # indicated by the output of the cfy status command. +# -- The plugins have been loaded. This is indicated by the +# presence of the /opt/manager/plugins-loaded file. -RULES_LOADED=/opt/manager/extra-resolver-rules-loaded +PLUGINS_LOADED=/opt/manager/plugins-loaded set -x -if [[ -f $RULES_LOADED ]] +if [[ -f $PLUGINS_LOADED ]] then # Check for all services running if /scripts/cloudify-ready.sh diff --git a/cm-container/scripts/set-resolver-rules.sh b/cm-container/scripts/set-resolver-rules.sh deleted file mode 100755 index e695231..0000000 --- a/cm-container/scripts/set-resolver-rules.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# ============LICENSE_START======================================================= -# org.onap.dcae -# ================================================================================ -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= - -set -ex -EXTRA_RULES=/opt/manager/extra-resolver-rules -PY=/opt/manager/env/bin/python -# Wait for Cloudify Manager to come up -while ! /scripts/cloudify-ready.sh -do - echo "Waiting for CM to come up" - sleep 15 -done - -if [[ -s ${EXTRA_RULES} && -r ${EXTRA_RULES} ]] -then - # Capture current resolver rules and append to new rules - ${PY} /scripts/update_resolver.py --dry-run | egrep "^-" >> ${EXTRA_RULES} - - # Update the resolver rules - ${PY} /scripts/update_resolver.py ${EXTRA_RULES} - systemctl restart cloudify-restservice.service - mv ${EXTRA_RULES} ${EXTRA_RULES}-loaded -fi
\ No newline at end of file diff --git a/cm-container/scripts/update_resolver.py b/cm-container/scripts/update_resolver.py deleted file mode 100644 index e5c9d46..0000000 --- a/cm-container/scripts/update_resolver.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -#============LICENSE_START========================================================== -# org.onap.dcae -# ================================================================================== -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. -# ================================================================================== -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END=========================================================== -# -import sys -import yaml -from sqlalchemy.orm.attributes import flag_modified -from manager_rest.flask_utils import setup_flask_app -from manager_rest.constants import PROVIDER_CONTEXT_ID -from manager_rest.storage import get_storage_manager, models - - -def main(dry_run, rules_file): - - with setup_flask_app().app_context(): - sm = get_storage_manager() - ctx = sm.get(models.ProviderContext, PROVIDER_CONTEXT_ID) - print 'Resolver rules before update:' - print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules']) - - if dry_run: - return - - with open(rules_file, 'r') as rules: - new_rules = yaml.load(rules) - ctx.context['cloudify']['import_resolver']['parameters']['rules'] = new_rules - print '\nResolver rules to update:' - print yaml.safe_dump(new_rules) - flag_modified(ctx, 'context') - sm.update(ctx) - print '\nProvide Context Saved' - print '\nResolver rules after update:' - print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules']) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print 'Must provide path to yaml file containing new rules or --dry-run' - exit(1) - - main(sys.argv[1]=='--dry-run', sys.argv[1])
\ No newline at end of file |