Move plugin upload to CM container

Issue-ID: DCAEGEN2-2236
Issue-ID: DCAEGEN2-1789
Signed-off-by: Jack Lucas <jflucas@research.att.com>
Change-Id: Ibc4612dedbf73a17a237b88abe198add391b82de

9 files changed, 249 insertions, 191 deletions

diff --git a/cm-container/Dockerfile-template b/cm-container/Dockerfile-template
index b392e92..5e6d846 100644
--- a/cm-container/Dockerfile-template
+++ b/cm-container/Dockerfile-template
@@ -1,7 +1,5 @@
 # ============LICENSE_START=======================================================
-# org.onap.dcae
-# ================================================================================
-# Copyright (c) 2018-2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,27 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============LICENSE_END=========================================================
-#
-# ECOMP is a trademark and service mark of AT&T Intellectual Property.
 FROM cloudifyplatform/community:19.01.24
-MAINTAINER maintainer
 
-ENV TYPE_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_plugins_releases }}
-ENV CCSDK_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releases }}
+ENV PLUGIN_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_plugins_releases }}
 
-# Store type files locally
+# Store plugin files locally
 RUN mkdir scripts
 COPY scripts/* scripts/
-# Load our type files and the Cloudify 3.4 type files
+# Load our plugin files
 # Setup rc.local to set up k8s credentials for CM
-RUN scripts/get-type-files.sh ${TYPE_REPO} ${CCSDK_REPO}\
-    && mkdir /opt/manager/resources/spec/cloudify/3.4\
-    && curl -Ss -L -f https://getcloudify.org/spec/cloudify/3.4/types.yaml > /opt/manager/resources/spec/cloudify/3.4/types.yaml\
-    && chown -R cfyuser:cfyuser /opt/manager/resources/spec/cloudify/3.4\
+RUN scripts/get-plugins.sh ${PLUGIN_REPO} \
     && chmod +x scripts/*.sh\
     && /scripts/configure-tls.sh\
     && echo "/scripts/setup-secret.sh" >> /etc/rc.d/rc.local\
-    && echo "/scripts/set-resolver-rules.sh" >> /etc/rc.d/rc.local\
+    && echo "/scripts/load-plugins.sh > /load-plugins.log 2>&1" >> /etc/rc.d/rc.local\
     && chmod +x /etc/rc.d/rc.local
 # Create mount point for CM config file
 RUN mkdir -p /opt/onap && chown cfyuser:cfyuser /opt/onap
diff --git a/cm-container/findPlugins.sh b/cm-container/findPlugins.sh
new file mode 100755
index 0000000..10d49a7
--- /dev/null
+++ b/cm-container/findPlugins.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (c)2020 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# Use the Nexus API to get a list of all the plugins/typefiles currently in Nexus
+# under a specified root directory, passed as the first argument to the script
+# We assume the repo structure that the build process uses:
+# PLUGIN_ROOT/<plugin_name>/<version>/<wagon_name>.wgn
+# PLUGIN_ROOT/<plugin_name>/<version>/<type_file_name>.yaml
+
+# This code could be used as the basis for an alternative to the existing
+# 'get-plugins.sh' script.  Instead of pulling a hard-coded list of plugins and
+# type files from Nexus, it would pull all of the plugins and type files, in all
+# available versions, from the Nexus repo.
+
+# At the very least, it is a useful tool for finding out what plugins and
+# type files have been loaded into the Nexus repo and are therefore available
+# to be included in the hard-coded list.
+
+shopt -s expand_aliases
+
+alias cu='curl -Ss -H "Accept: application/json" -L -f'
+PLUGIN_ROOT=${1:-"https://nexus.onap.org/service/local/repositories/raw/content/org.onap.dcaegen2.platform.plugins/R7/"}
+
+function getPlugins() {
+    local root=$1
+    # Get URLs for all of the plugins under the plugin root directory
+    local PLUGINS=$(cu $root | jq .data[].resourceURI | sed -e 's/"//g')
+
+    # Go into each plugin directory
+    for p in $PLUGINS
+    do
+        # Get the available versions of the plugin
+        local VERSIONS=$(cu $p | jq .data[].resourceURI | sed -e 's/"//g')
+
+        # Get wagon and type file for each version
+        for v in $VERSIONS
+        do
+            local RESOURCES=$(cu $v | jq .data[].relativePath)
+            # RESOURCES will have a list of everything in the version directory, including many timestamped
+	          # wagons and type files, and some zip files as well.  For each version, there should be a single
+            # non-time-stamped .wgn and a single non-timestamped .yaml file.   We try to pull these from the
+            # from the list using grep, and then we reformat the results to remove quote marks and to remove
+            # the first two levels of the relative path.
+            # Just in case we're wrong about how many non-timestamped .wgn and .yaml files are in each directory,
+            # we treat the result of the grep as an array and we take the first element only.
+	          local w=($((for r in $RESOURCES; do echo $r; done) | grep ".wgn\"" | tr -d '"' | sed -e 's#^/[^/]*/[^/]*##'))
+	          local t=($((for r in $RESOURCES; do echo $r; done) | grep ".yaml\"" | tr -d '"' | sed -e 's#^/[^/]*/[^/]*##'))
+            echo "${w[0]}|${t[0]}"
+            # We could potentially fetch the plugin wagon file and the type file here.  Probably wouldn't need
+            # to have this code as a function in that case.  Also, probably should the .resourceURI (the full URL
+            # of each file) rather than .relativePath, and not strip off any part of the path (the 'sed' operation
+            # would not be needed.)
+        done
+    done
+}
+
+for p in $(getPlugins $PLUGIN_ROOT | sort)
+do
+  echo $p
+done
diff --git a/cm-container/pom.xml b/cm-container/pom.xml
index 207e979..4da4d8c 100644
--- a/cm-container/pom.xml
+++ b/cm-container/pom.xml
@@ -27,7 +27,7 @@ limitations under the License.
   <groupId>org.onap.dcaegen2.deployments</groupId>
   <artifactId>cm-container</artifactId>
   <name>dcaegen2-deployments-cm-container</name>
-  <version>2.1.0</version>
+  <version>3.0.0</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/cm-container/scripts/get-plugins.sh b/cm-container/scripts/get-plugins.sh
new file mode 100755
index 0000000..7c2f353
--- /dev/null
+++ b/cm-container/scripts/get-plugins.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# Pull plugin wagon files and type files from repo
+# $1 is the DCAE repo URL - script assumes all files come from the
+#    same repo, but not necessarily same paths
+#
+set -x -e
+
+# Location in CM container where plugins/type files will be stored
+# At deployment, CM script will look here to find plugins to upload
+DEST=${DEST:-/opt/plugins}
+
+# Each line has a plugin wagon/type file pair, in the form
+# /path/to/plugin/wagon|/path/to/type/file
+PLUGINS=\
+"\
+/dcaepolicyplugin/2.4.0/dcaepolicyplugin-2.4.0-py27-none-linux_x86_64.wgn|/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
+/relationshipplugin/1.1.0/relationshipplugin-1.1.0-py27-none-linux_x86_64.wgn|/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
+/k8splugin/2.0.0/k8splugin-2.0.0-py27-none-linux_x86_64.wgn|/k8splugin/2.0.0/k8splugin_types.yaml \
+/k8splugin/3.0.0/k8splugin-3.0.0-py27-none-linux_x86_64.wgn|/k8splugin/3.0.0/k8splugin_types.yaml \
+/clamppolicyplugin/1.1.0/clamppolicyplugin-1.1.0-py27-none-linux_x86_64.wgn|/clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
+/dmaap/1.5.0/dmaap-1.5.0-py27-none-linux_x86_64.wgn|/dmaap/1.5.0/dmaap_types.yaml \
+/helm/4.2.0/helm-4.2.0-py27-none-linux_x86_64.wgn|/helm/4.2.0/helm_types.yaml \
+/pgaas/1.3.0/pgaas-1.3.0-py27-none-linux_x86_64.wgn|/pgaas/1.3.0/pgaas_types.yaml \
+/sshkeyshare/1.2.0/sshkeyshare-1.2.0-py27-none-linux_x86_64.wgn|/sshkeyshare/1.2.0/sshkeyshare_types.yaml
+"
+
+mkdir -p ${DEST}
+
+for p in ${PLUGINS}
+do
+  w=$(echo $p | cut -d '|' -f1)
+  t=$(echo $p | cut -d '|' -f2)
+
+  # Put each wagon/type file pair into its own subdirectory
+  # This prevents name collisions which can happen because
+  # type files don't embed a version.
+	subdir=$(mktemp -d -t plugin-XXXXXXX --tmpdir=${DEST})
+
+	curl -Ss -L -f $1/$t >> ${subdir}/$(basename $t)
+  curl -Ss -L -f $1/$w >> ${subdir}/$(basename $w)
+
+done
+
+chown -R cfyuser:cfyuser ${DEST}
diff --git a/cm-container/scripts/get-type-files.sh b/cm-container/scripts/get-type-files.sh
deleted file mode 100755
index 3b25bbb..0000000
--- a/cm-container/scripts/get-type-files.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-# ============LICENSE_START=======================================================
-# org.onap.dcae
-# ================================================================================
-# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END=========================================================
-
-# Pull type files from repos
-# Set up the CM import resolver
-# $1 is the DCAE repo URL
-# $2 is the CCSDK repo URL
-#
-set -x
-DEST=/opt/manager/resources/onapspec
-EXTRA_RULES=/opt/manager/extra-resolver-rules
-
-DCAETYPEFILES=\
-"\
-/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
-/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
-/k8splugin/1.7.2/k8splugin_types.yaml \
-/k8splugin/2.0.0/k8splugin_types.yaml \
-clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
-
-"
-
-CCSDKTYPEFILES=\
-"\
-/type_files/pgaas/1.1.0/pgaas_types.yaml \
-/type_files/sshkeyshare/sshkey_types.yaml \
-/type_files/helm/4.0.2/helm-type.yaml \
-/type_files/dmaap/dmaap.yaml \
-"
-
-mkdir ${DEST}
-
-for typefile in ${DCAETYPEFILES}
-do
-	mkdir -p ${DEST}/$(dirname ${typefile})
-	curl -Ss -L -f $1/${typefile} >> ${DEST}/${typefile}
-done
-
-for typefile in ${CCSDKTYPEFILES}
-do
-	mkdir -p ${DEST}/$(dirname ${typefile})
-	curl -Ss -L -f $2/${typefile} >> ${DEST}/${typefile}
-done
-
-chown cfyuser:cfyuser ${DEST}
-
-# Add our local type file store to CM import resolver configuration
-TYPE_RULE0="{\"$1\": \"file://${DEST}\"}"
-TYPE_RULE1="{\"$2\": \"file://${DEST}\"}"
-# This sed re is 'brittle' but we can be sure the config.yaml file
-# won't change as long as we do not change the source Docker image for CM
-sed -i -e "s#      rules:#      rules:\n      - ${TYPE_RULE0}#" /etc/cloudify/config.yaml
-sed -i -e "s#      rules:#      rules:\n      - ${TYPE_RULE1}#" /etc/cloudify/config.yaml
-
-chown cfyuser:cfyuser /etc/cloudify/config.yaml
-
-# Changing /etc/cloudify/config.yaml is no longer sufficient
-# Need to provide the additional rules in a file that can be
-# used at deployment time to update the resolver rules
-echo "- ${TYPE_RULE0}" > ${EXTRA_RULES}
-echo "- ${TYPE_RULE1}" >> ${EXTRA_RULES}
diff --git a/cm-container/scripts/load-plugins.sh b/cm-container/scripts/load-plugins.sh
new file mode 100755
index 0000000..f4d1f66
--- /dev/null
+++ b/cm-container/scripts/load-plugins.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+# Runs at deployment time to load the plugins/type files
+# that were stored into the container file system at build time
+
+PLUGINS_LOADED=/opt/manager/plugins-loaded
+PLUGIN_DIR=/opt/plugins
+
+# Set defaults for CM address, protocol, and port
+CMADDR=${CMADDR:-dcae-cloudify-manager}
+CMPROTO=${CMPROTO:-https}
+CMPORT=${CMPORT:-443}
+
+# Password is currently fixed at the default
+# Eventually the password will be passed in as an environment variable
+CMPASS=${CMPASS:-admin}
+
+# Set up additional parameters for using HTTPS
+CACERT="/opt/onap/certs/cacert.pem"
+CURLTLS=""
+if [ $CMPROTO = "https" ]
+then
+    CURLTLS="--cacert $CACERT"
+fi
+
+### FUNCTION DEFINITIONS ###
+
+# cm_hasany: Query Cloudify Manager and return 0 (true) if there are any entities matching the query
+# Used to see if something is already present on CM
+# $1 -- query fragment, for instance "plugins?archive_name=xyz.wgn" to get
+#  the number of plugins that came from the archive file "xyz.wgn"
+function cm_hasany {
+    # We use _include=id to limit the amount of data the CM sends back
+    # We rely on the "metadata.pagination.total" field in the response
+    # for the total number of matching entities
+    COUNT=$(curl -Ss -H "Tenant: default_tenant" --user admin:${CMPASS} ${CURLTLS} "${CMPROTO}://${CMADDR}:${CMPORT}/api/v3.1/$1&_include=id" \
+             | /bin/jq .metadata.pagination.total)
+    if (( $COUNT > 0 ))
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Install plugin if it's not already installed
+# $1 -- path to wagon file for plugin
+# $2 -- path to type file for plugin
+function install_plugin {
+    ARCHIVE=$(basename $1)
+    # See if it's already installed
+    if cm_hasany "plugins?archive_name=$ARCHIVE"
+    then
+        echo plugin $1 already installed on ${CMADDR}
+    else
+        cfy plugin upload  -y $2 $1
+    fi
+}
+
+### END FUNCTION DEFINTIONS ###
+
+set -ex
+
+
+# Wait for Cloudify Manager to come up
+while ! /scripts/cloudify-ready.sh
+do
+    echo "Waiting for CM to come up"
+    sleep 15
+done
+
+if [[ ! -f ${PLUGINS_LOADED} ]]
+then
+
+  # Each subdirectory of ${PLUGIN_DIR} contains a wagon (.wgn) and type file (.yaml)
+  for p in ${PLUGIN_DIR}/*
+  do
+    # Expecting exactly 1 .wgn and 1 .yaml
+    # But just in case, taking only the first of each
+    # (If either is missing, will fail on install_plugin)
+    wagons=($p/*.wgn)
+    types=($p/*.yaml)
+    install_plugin ${wagons[0]} ${types[0]}
+  done
+
+  touch ${PLUGINS_LOADED}
+else
+  echo "Plugins already loaded"
+fi
\ No newline at end of file
diff --git a/cm-container/scripts/readiness-check.sh b/cm-container/scripts/readiness-check.sh
index a7e92ab..d706c0d 100644
--- a/cm-container/scripts/readiness-check.sh
+++ b/cm-container/scripts/readiness-check.sh
@@ -2,7 +2,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,12 +24,14 @@
 #       /opt/manager/extra-resolver-rules-loaded.
 #    -- All Cloudify Manager services must be running, as
 #       indicated by the output of the cfy status command.
+#    -- The plugins have been loaded.  This is indicated by the
+#       presence of the /opt/manager/plugins-loaded file.
 
-RULES_LOADED=/opt/manager/extra-resolver-rules-loaded
+PLUGINS_LOADED=/opt/manager/plugins-loaded
 
 set -x
 
-if [[ -f  $RULES_LOADED ]]
+if [[ -f  $PLUGINS_LOADED ]]
 then
   # Check for all services running
   if /scripts/cloudify-ready.sh
diff --git a/cm-container/scripts/set-resolver-rules.sh b/cm-container/scripts/set-resolver-rules.sh
deleted file mode 100755
index e695231..0000000
--- a/cm-container/scripts/set-resolver-rules.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-# ============LICENSE_START=======================================================
-# org.onap.dcae
-# ================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END=========================================================
-
-set -ex
-EXTRA_RULES=/opt/manager/extra-resolver-rules
-PY=/opt/manager/env/bin/python
-# Wait for Cloudify Manager to come up
-while ! /scripts/cloudify-ready.sh
-do
-    echo "Waiting for CM to come up"
-    sleep 15
-done
-
-if [[ -s ${EXTRA_RULES} && -r ${EXTRA_RULES} ]]
-then
-    # Capture current resolver rules and append to new rules
-    ${PY} /scripts/update_resolver.py --dry-run | egrep "^-" >> ${EXTRA_RULES}
-
-    # Update the resolver rules
-    ${PY} /scripts/update_resolver.py ${EXTRA_RULES}
-    systemctl restart cloudify-restservice.service
-    mv ${EXTRA_RULES} ${EXTRA_RULES}-loaded
-fi
\ No newline at end of file
diff --git a/cm-container/scripts/update_resolver.py b/cm-container/scripts/update_resolver.py
deleted file mode 100644
index e5c9d46..0000000
--- a/cm-container/scripts/update_resolver.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-#============LICENSE_START==========================================================
-# org.onap.dcae
-# ==================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
-# ==================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END===========================================================
-#
-import sys
-import yaml
-from sqlalchemy.orm.attributes import flag_modified
-from manager_rest.flask_utils import setup_flask_app
-from manager_rest.constants import PROVIDER_CONTEXT_ID
-from manager_rest.storage import get_storage_manager, models
-
-
-def main(dry_run, rules_file):
-
-    with setup_flask_app().app_context():
-        sm = get_storage_manager()
-        ctx = sm.get(models.ProviderContext, PROVIDER_CONTEXT_ID)
-        print 'Resolver rules before update:'
-        print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules'])
-
-        if dry_run:
-            return
-
-        with open(rules_file, 'r') as rules:
-            new_rules = yaml.load(rules)
-        ctx.context['cloudify']['import_resolver']['parameters']['rules'] = new_rules
-        print '\nResolver rules to update:'
-        print yaml.safe_dump(new_rules)
-        flag_modified(ctx, 'context')
-        sm.update(ctx)
-        print '\nProvide Context Saved'
-        print '\nResolver rules after update:'
-        print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules'])
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print 'Must provide path to yaml file containing new rules or --dry-run'
-        exit(1)
-
-    main(sys.argv[1]=='--dry-run', sys.argv[1])
\ No newline at end of file