Move plugin upload to CM container

Issue-ID: DCAEGEN2-2236
Issue-ID: DCAEGEN2-1789
Signed-off-by: Jack Lucas <jflucas@research.att.com>
Change-Id: Ibc4612dedbf73a17a237b88abe198add391b82de

6 files changed, 168 insertions, 175 deletions

diff --git a/cm-container/scripts/get-plugins.sh b/cm-container/scripts/get-plugins.sh
new file mode 100755
index 0000000..7c2f353
--- /dev/null
+++ b/cm-container/scripts/get-plugins.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# Pull plugin wagon files and type files from repo
+# $1 is the DCAE repo URL - script assumes all files come from the
+#    same repo, but not necessarily same paths
+#
+set -x -e
+
+# Location in CM container where plugins/type files will be stored
+# At deployment, CM script will look here to find plugins to upload
+DEST=${DEST:-/opt/plugins}
+
+# Each line has a plugin wagon/type file pair, in the form
+# /path/to/plugin/wagon|/path/to/type/file
+PLUGINS=\
+"\
+/dcaepolicyplugin/2.4.0/dcaepolicyplugin-2.4.0-py27-none-linux_x86_64.wgn|/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
+/relationshipplugin/1.1.0/relationshipplugin-1.1.0-py27-none-linux_x86_64.wgn|/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
+/k8splugin/2.0.0/k8splugin-2.0.0-py27-none-linux_x86_64.wgn|/k8splugin/2.0.0/k8splugin_types.yaml \
+/k8splugin/3.0.0/k8splugin-3.0.0-py27-none-linux_x86_64.wgn|/k8splugin/3.0.0/k8splugin_types.yaml \
+/clamppolicyplugin/1.1.0/clamppolicyplugin-1.1.0-py27-none-linux_x86_64.wgn|/clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
+/dmaap/1.5.0/dmaap-1.5.0-py27-none-linux_x86_64.wgn|/dmaap/1.5.0/dmaap_types.yaml \
+/helm/4.2.0/helm-4.2.0-py27-none-linux_x86_64.wgn|/helm/4.2.0/helm_types.yaml \
+/pgaas/1.3.0/pgaas-1.3.0-py27-none-linux_x86_64.wgn|/pgaas/1.3.0/pgaas_types.yaml \
+/sshkeyshare/1.2.0/sshkeyshare-1.2.0-py27-none-linux_x86_64.wgn|/sshkeyshare/1.2.0/sshkeyshare_types.yaml
+"
+
+mkdir -p ${DEST}
+
+for p in ${PLUGINS}
+do
+  w=$(echo $p | cut -d '|' -f1)
+  t=$(echo $p | cut -d '|' -f2)
+
+  # Put each wagon/type file pair into its own subdirectory
+  # This prevents name collisions which can happen because
+  # type files don't embed a version.
+	subdir=$(mktemp -d -t plugin-XXXXXXX --tmpdir=${DEST})
+
+	curl -Ss -L -f $1/$t >> ${subdir}/$(basename $t)
+  curl -Ss -L -f $1/$w >> ${subdir}/$(basename $w)
+
+done
+
+chown -R cfyuser:cfyuser ${DEST}
diff --git a/cm-container/scripts/get-type-files.sh b/cm-container/scripts/get-type-files.sh
deleted file mode 100755
index 3b25bbb..0000000
--- a/cm-container/scripts/get-type-files.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-# ============LICENSE_START=======================================================
-# org.onap.dcae
-# ================================================================================
-# Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END=========================================================
-
-# Pull type files from repos
-# Set up the CM import resolver
-# $1 is the DCAE repo URL
-# $2 is the CCSDK repo URL
-#
-set -x
-DEST=/opt/manager/resources/onapspec
-EXTRA_RULES=/opt/manager/extra-resolver-rules
-
-DCAETYPEFILES=\
-"\
-/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
-/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
-/k8splugin/1.7.2/k8splugin_types.yaml \
-/k8splugin/2.0.0/k8splugin_types.yaml \
-clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
-
-"
-
-CCSDKTYPEFILES=\
-"\
-/type_files/pgaas/1.1.0/pgaas_types.yaml \
-/type_files/sshkeyshare/sshkey_types.yaml \
-/type_files/helm/4.0.2/helm-type.yaml \
-/type_files/dmaap/dmaap.yaml \
-"
-
-mkdir ${DEST}
-
-for typefile in ${DCAETYPEFILES}
-do
-	mkdir -p ${DEST}/$(dirname ${typefile})
-	curl -Ss -L -f $1/${typefile} >> ${DEST}/${typefile}
-done
-
-for typefile in ${CCSDKTYPEFILES}
-do
-	mkdir -p ${DEST}/$(dirname ${typefile})
-	curl -Ss -L -f $2/${typefile} >> ${DEST}/${typefile}
-done
-
-chown cfyuser:cfyuser ${DEST}
-
-# Add our local type file store to CM import resolver configuration
-TYPE_RULE0="{\"$1\": \"file://${DEST}\"}"
-TYPE_RULE1="{\"$2\": \"file://${DEST}\"}"
-# This sed re is 'brittle' but we can be sure the config.yaml file
-# won't change as long as we do not change the source Docker image for CM
-sed -i -e "s#      rules:#      rules:\n      - ${TYPE_RULE0}#" /etc/cloudify/config.yaml
-sed -i -e "s#      rules:#      rules:\n      - ${TYPE_RULE1}#" /etc/cloudify/config.yaml
-
-chown cfyuser:cfyuser /etc/cloudify/config.yaml
-
-# Changing /etc/cloudify/config.yaml is no longer sufficient
-# Need to provide the additional rules in a file that can be
-# used at deployment time to update the resolver rules
-echo "- ${TYPE_RULE0}" > ${EXTRA_RULES}
-echo "- ${TYPE_RULE1}" >> ${EXTRA_RULES}
diff --git a/cm-container/scripts/load-plugins.sh b/cm-container/scripts/load-plugins.sh
new file mode 100755
index 0000000..f4d1f66
--- /dev/null
+++ b/cm-container/scripts/load-plugins.sh
@@ -0,0 +1,103 @@
+#!/bin/bash
+# ============LICENSE_START=======================================================
+# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+# Runs at deployment time to load the plugins/type files
+# that were stored into the container file system at build time
+
+PLUGINS_LOADED=/opt/manager/plugins-loaded
+PLUGIN_DIR=/opt/plugins
+
+# Set defaults for CM address, protocol, and port
+CMADDR=${CMADDR:-dcae-cloudify-manager}
+CMPROTO=${CMPROTO:-https}
+CMPORT=${CMPORT:-443}
+
+# Password is currently fixed at the default
+# Eventually the password will be passed in as an environment variable
+CMPASS=${CMPASS:-admin}
+
+# Set up additional parameters for using HTTPS
+CACERT="/opt/onap/certs/cacert.pem"
+CURLTLS=""
+if [ $CMPROTO = "https" ]
+then
+    CURLTLS="--cacert $CACERT"
+fi
+
+### FUNCTION DEFINITIONS ###
+
+# cm_hasany: Query Cloudify Manager and return 0 (true) if there are any entities matching the query
+# Used to see if something is already present on CM
+# $1 -- query fragment, for instance "plugins?archive_name=xyz.wgn" to get
+#  the number of plugins that came from the archive file "xyz.wgn"
+function cm_hasany {
+    # We use _include=id to limit the amount of data the CM sends back
+    # We rely on the "metadata.pagination.total" field in the response
+    # for the total number of matching entities
+    COUNT=$(curl -Ss -H "Tenant: default_tenant" --user admin:${CMPASS} ${CURLTLS} "${CMPROTO}://${CMADDR}:${CMPORT}/api/v3.1/$1&_include=id" \
+             | /bin/jq .metadata.pagination.total)
+    if (( $COUNT > 0 ))
+    then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Install plugin if it's not already installed
+# $1 -- path to wagon file for plugin
+# $2 -- path to type file for plugin
+function install_plugin {
+    ARCHIVE=$(basename $1)
+    # See if it's already installed
+    if cm_hasany "plugins?archive_name=$ARCHIVE"
+    then
+        echo plugin $1 already installed on ${CMADDR}
+    else
+        cfy plugin upload  -y $2 $1
+    fi
+}
+
+### END FUNCTION DEFINTIONS ###
+
+set -ex
+
+
+# Wait for Cloudify Manager to come up
+while ! /scripts/cloudify-ready.sh
+do
+    echo "Waiting for CM to come up"
+    sleep 15
+done
+
+if [[ ! -f ${PLUGINS_LOADED} ]]
+then
+
+  # Each subdirectory of ${PLUGIN_DIR} contains a wagon (.wgn) and type file (.yaml)
+  for p in ${PLUGIN_DIR}/*
+  do
+    # Expecting exactly 1 .wgn and 1 .yaml
+    # But just in case, taking only the first of each
+    # (If either is missing, will fail on install_plugin)
+    wagons=($p/*.wgn)
+    types=($p/*.yaml)
+    install_plugin ${wagons[0]} ${types[0]}
+  done
+
+  touch ${PLUGINS_LOADED}
+else
+  echo "Plugins already loaded"
+fi
\ No newline at end of file
diff --git a/cm-container/scripts/readiness-check.sh b/cm-container/scripts/readiness-check.sh
index a7e92ab..d706c0d 100644
--- a/cm-container/scripts/readiness-check.sh
+++ b/cm-container/scripts/readiness-check.sh
@@ -2,7 +2,7 @@
 # ============LICENSE_START=======================================================
 # org.onap.dcae
 # ================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,12 +24,14 @@
 #       /opt/manager/extra-resolver-rules-loaded.
 #    -- All Cloudify Manager services must be running, as
 #       indicated by the output of the cfy status command.
+#    -- The plugins have been loaded.  This is indicated by the
+#       presence of the /opt/manager/plugins-loaded file.
 
-RULES_LOADED=/opt/manager/extra-resolver-rules-loaded
+PLUGINS_LOADED=/opt/manager/plugins-loaded
 
 set -x
 
-if [[ -f  $RULES_LOADED ]]
+if [[ -f  $PLUGINS_LOADED ]]
 then
   # Check for all services running
   if /scripts/cloudify-ready.sh
diff --git a/cm-container/scripts/set-resolver-rules.sh b/cm-container/scripts/set-resolver-rules.sh
deleted file mode 100755
index e695231..0000000
--- a/cm-container/scripts/set-resolver-rules.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-# ============LICENSE_START=======================================================
-# org.onap.dcae
-# ================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END=========================================================
-
-set -ex
-EXTRA_RULES=/opt/manager/extra-resolver-rules
-PY=/opt/manager/env/bin/python
-# Wait for Cloudify Manager to come up
-while ! /scripts/cloudify-ready.sh
-do
-    echo "Waiting for CM to come up"
-    sleep 15
-done
-
-if [[ -s ${EXTRA_RULES} && -r ${EXTRA_RULES} ]]
-then
-    # Capture current resolver rules and append to new rules
-    ${PY} /scripts/update_resolver.py --dry-run | egrep "^-" >> ${EXTRA_RULES}
-
-    # Update the resolver rules
-    ${PY} /scripts/update_resolver.py ${EXTRA_RULES}
-    systemctl restart cloudify-restservice.service
-    mv ${EXTRA_RULES} ${EXTRA_RULES}-loaded
-fi
\ No newline at end of file
diff --git a/cm-container/scripts/update_resolver.py b/cm-container/scripts/update_resolver.py
deleted file mode 100644
index e5c9d46..0000000
--- a/cm-container/scripts/update_resolver.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-#============LICENSE_START==========================================================
-# org.onap.dcae
-# ==================================================================================
-# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
-# ==================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============LICENSE_END===========================================================
-#
-import sys
-import yaml
-from sqlalchemy.orm.attributes import flag_modified
-from manager_rest.flask_utils import setup_flask_app
-from manager_rest.constants import PROVIDER_CONTEXT_ID
-from manager_rest.storage import get_storage_manager, models
-
-
-def main(dry_run, rules_file):
-
-    with setup_flask_app().app_context():
-        sm = get_storage_manager()
-        ctx = sm.get(models.ProviderContext, PROVIDER_CONTEXT_ID)
-        print 'Resolver rules before update:'
-        print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules'])
-
-        if dry_run:
-            return
-
-        with open(rules_file, 'r') as rules:
-            new_rules = yaml.load(rules)
-        ctx.context['cloudify']['import_resolver']['parameters']['rules'] = new_rules
-        print '\nResolver rules to update:'
-        print yaml.safe_dump(new_rules)
-        flag_modified(ctx, 'context')
-        sm.update(ctx)
-        print '\nProvide Context Saved'
-        print '\nResolver rules after update:'
-        print yaml.safe_dump(ctx.context['cloudify']['import_resolver']['parameters']['rules'])
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print 'Must provide path to yaml file containing new rules or --dry-run'
-        exit(1)
-
-    main(sys.argv[1]=='--dry-run', sys.argv[1])
\ No newline at end of file