[DCAE] Update Cloudify Manager to Python 3.x4.0.0-cm-container

Issue-ID: DCAEGEN2-2542
Signed-off-by: Jack Lucas <jflos@sonoris.net>
Change-Id: I8d3c42fc80290394df4eb3bd724c6e43d60c0a96

8 files changed, 47 insertions, 108 deletions

diff --git a/cm-container/Dockerfile-template b/cm-container/Dockerfile-template
index b5ce5ab..c794cff 100644
--- a/cm-container/Dockerfile-template
+++ b/cm-container/Dockerfile-template
@@ -1,6 +1,6 @@
 # ============LICENSE_START=======================================================
 # Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
-# Copyright (c) 2020 J. F. Lucas.  All rights reserved.
+# Copyright (c) 2020-2021 J. F. Lucas.  All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,23 +14,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============LICENSE_END=========================================================
-FROM cloudifyplatform/community-cloudify-manager-aio:20.03.03
+FROM cloudifyplatform/community-cloudify-manager-aio:20.12.15
 
 ENV PLUGIN_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2_platform_plugins_releases }}
 
-# Store plugin files locally
+# Install openssh-clients to ssh-keygen, used by the sshkeyshare plugin
+RUN yum install -y openssh-clients
+
 RUN mkdir scripts
 COPY scripts/* scripts/
-COPY reset_admin.py /opt/manager/
 
-# Load plugins, set up TLS configuration, load Cloudify type file
-RUN chmod +x /opt/manager/reset_admin.py \
-    && chmod +x scripts/*.sh \
-    && scripts/get-plugins.sh ${PLUGIN_REPO} \
-    && /scripts/configure-tls.sh \
-    && mkdir /opt/manager/resources/spec/cloudify/4.5.5 \
-    && curl -Ss -L -f https://www.getcloudify.org/spec/cloudify/4.5.5/types.yaml > /opt/manager/resources/spec/cloudify/4.5.5/types.yaml \
-    && chown -R cfyuser:cfyuser /opt/manager/resources/spec/cloudify/4.5.5
+# Load plugins and set up TLS configuration
+  RUN chmod +x scripts/*.sh \
+  && scripts/get-plugins.sh ${PLUGIN_REPO} \
+  && /scripts/configure-tls.sh
 
 # Create mount point for CM config file
 RUN mkdir -p /opt/onap && chown cfyuser:cfyuser /opt/onap
diff --git a/cm-container/README.md b/cm-container/README.md
index 03b9634..b929e21 100644
--- a/cm-container/README.md
+++ b/cm-container/README.md
@@ -1,13 +1,17 @@
-# Cloudify Manager Container Builder
+# Cloudify Manager Image Builder
 ## Purpose
 The artifacts in this directory build a Docker image based on the
-public image from Cloudify (`cloudifyplatform/community`).  The
-image has the Cloudify Manager software from the base image
-and adds our types files.  It configures
-the import resolver to use our local type files instead
-of fetching them over the Internet.   It sets up the `/opt/onap` mount point
-for our config files.  It also sets up the certificate, key and other
-configuration for using TLS.
+public image from Cloudify (`cloudifyplatform/community`).  The image
+build process, driven by the Dockerfile:
+  - retrieves the Cloudify plugins and type files needed to deploy DCAE components.  
+  - edits the Cloudify configuration file (`/etc/cloudify/config.yaml`) to
+set up Cloudify Manager to use TLS on its HTTP interfaces.
+  - sets up the `/opt/onap` mount point for configuration file.  
+  - installs scripts that run when the container is started.  These scripts:
+      - set up persistent storage for the container
+      - make the credentials for accessing the Kubernetes API available to Cloudify Manager
+      - set the administrative password to a value generated during the Helm deployment process
+      - upload the plugins and type files to the Cloudify Manager once it is running
 
 ## Running the Container
 The container is intended to be launched via a Helm chart as part
@@ -60,8 +64,6 @@ which then brings up the many other processes needed for a working instance of C
 ## The `setup-secret.sh` script
 When Kubernetes starts a container, it mounts a directory containing the credentials that the container needs to access the Kubernetes API on the local Kubernetes cluster.  The mountpoint is `/var/run/secrets/kubernetes.io/serviceaccount`.   Something about the way that Cloudify Manager is started (possibly because `/sbin/init` is run) causes this mountpoint to be hidden.   `setup-secret.sh` will recreate the directory if it's not present and symbolically link it to a copy of the credentials mounted at `/secret` in the container file system.  This gives Cloudify Manager the credentials that the Kubernetes plugin needs to deploy Kubernetes-based DCAE components.
 
-`setup-secret.sh` needs to run after '/sbin/init'.  The Dockerfile installs it in the `rc.local` script that runs at startup.
-
 ## Cleaning up Kubernetes components deployed by Cloudify Manager
 Using the `helm undeploy` (or `helm delete`) command will destroy the Kubernetes components deployed via helm.  In an ONAP deployment
 driven by OOM, this includes destroying Cloudify Manager.  helm will *not* delete Kubernetes components deployed by Cloudify Manager.
diff --git a/cm-container/pom.xml b/cm-container/pom.xml
index e676be6..b128ec3 100644
--- a/cm-container/pom.xml
+++ b/cm-container/pom.xml
@@ -2,6 +2,7 @@
 <!--
 ================================================================================
 Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
+Copyright (c) 2021 J. F. Lucas. All rights reserved.
 ================================================================================
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -27,7 +28,7 @@ limitations under the License.
   <groupId>org.onap.dcaegen2.deployments</groupId>
   <artifactId>cm-container</artifactId>
   <name>dcaegen2-deployments-cm-container</name>
-  <version>3.4.2</version>
+  <version>4.0.0</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/cm-container/reset_admin.py b/cm-container/reset_admin.py
deleted file mode 100644
index 2253341..0000000
--- a/cm-container/reset_admin.py
+++ /dev/null
@@ -1,54 +0,0 @@
-########
-# Copyright (c) 2018 Cloudify Platform Ltd. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    * See the License for the specific language governing permissions and
-#    * limitations under the License.
-
-#!/usr/bin/env python
-
-import subprocess
-import json
-import argparse
-from flask_security.utils import encrypt_password
-from manager_rest.flask_utils import setup_flask_app
-
-
-def db_update_password(password):
-    password = encrypt_new_password(password)
-    password = password.replace('$', '\$')
-    sql_command = "\"update users set password='" + password + "' where username='admin'\""
-    cmd = "sudo -u postgres psql cloudify_db -c " + sql_command
-    subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
-
-
-def get_salt():
-    with open('/opt/manager/rest-security.conf') as f:
-        rest_security = json.load(f)
-
-    return rest_security['hash_salt']
-
-
-def encrypt_new_password(password):
-    app = setup_flask_app(hash_salt=get_salt())
-    with app.app_context():
-        password = encrypt_password(password)
-    return password
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description=('Reset admin password in DB according to rest-security.conf'))
-    parser.add_argument('-p', '--password', required=True, help='New admin password')
-    args = parser.parse_args()
-
-    db_update_password(args.password)
-    print 'Password updated in DB!\n'
diff --git a/cm-container/scripts/configure-tls.sh b/cm-container/scripts/configure-tls.sh
index a4b817b..d7c9cc8 100644
--- a/cm-container/scripts/configure-tls.sh
+++ b/cm-container/scripts/configure-tls.sh
@@ -3,6 +3,7 @@
 # org.onap.dcae
 # ================================================================================
 # Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2020-2021 J. F. Lucas.  All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,26 +18,14 @@
 # limitations under the License.
 # ============LICENSE_END=========================================================
 #
-# Set up configuration files so that CM uses TLS on its external API
 
-# Change the nginx configuration -- this is what actually makes it work
-SSLCONFPATTERN="^include \"/etc/nginx/conf.d/http-external-rest-server.cloudify\""
-SSLCONFREPLACE="include \"/etc/nginx/conf.d/https-external-rest-server.cloudify\""
-sed -i -e "s#${SSLCONFPATTERN}#${SSLCONFREPLACE}#" /etc/nginx/conf.d/cloudify.conf
-
-# Set certificate and key locations
-sed -i -e "s#  ssl_certificate .*;#  ssl_certificate     /opt/onap/certs/cert.pem;#" /etc/nginx/conf.d/https-external-rest-server.cloudify
-sed -i -e "s#  ssl_certificate_key .*;#  ssl_certificate_key     /opt/onap/certs/key.pem;#" /etc/nginx/conf.d/https-external-rest-server.cloudify
-
-# Change the cloudify config file, just to be safe
-# Someone might run cfy_manager configure on the CM container for some reason
-# and we don't want them to overwrite the TLS configuration
-# (Running cfy_manager configure is a bad idea, though, because it often fails.)
+# Set tls to "enabled"
 sed -i -e "s#^    ssl_enabled: false#    ssl_enabled: true#" /etc/cloudify/config.yaml
-
-# The Cloudify command line tool ('cfy') needs to be configured for TLS as well
-# (The readiness check script uses 'cfy status')
-sed -i -e "s#^rest_port: 80#rest_port: 443#" /root/.cloudify/profiles/localhost/context
-sed -i -e "s/^rest_protocol: http$/rest_protocol: https/" /root/.cloudify/profiles/localhost/context
-sed -i -e "s#^rest_certificate: !!python/unicode '/etc/cloudify/ssl/cloudify_external_cert.pem'#rest_certificate: !!python/unicode '/opt/onap/certs/cacert.pem'#" /root/.cloudify/profiles/localhost/context
-sed -i -e "s#^manager_ip: !!python/unicode 'localhost'#manager_ip: !!python/unicode 'dcae-cloudify-manager'#" /root/.cloudify/profiles/localhost/context
+# Set up paths for our certificates
+sed -i -e "s|external_cert_path: .*$|external_cert_path: '/opt/onap/certs/cert.pem'|" /etc/cloudify/config.yaml
+sed -i -e "s|external_key_path: .*$|external_key_path: '/opt/onap/certs/key.pem'|" /etc/cloudify/config.yaml
+sed -i -e "s|external_ca_cert_path: .*$|external_ca_cert_path: '/opt/onap/certs/cacert.pem'|" /etc/cloudify/config.yaml
+# Set the host name for the local profile
+# Otherwise, the CM startup process will use 'localhost' and will fail
+# because the TLS certificate does not have 'localhost' as a CN or SAN
+sed -i -e 's/  cli_local_profile_host_name: .*$/  cli_local_profile_host_name: dcae-cloudify-manager/' /etc/cloudify/config.yaml
diff --git a/cm-container/scripts/get-plugins.sh b/cm-container/scripts/get-plugins.sh
index 5d11254..5162832 100755
--- a/cm-container/scripts/get-plugins.sh
+++ b/cm-container/scripts/get-plugins.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 # Copyright (c) 2018-2020 AT&T Intellectual Property. All rights reserved.
+# Copyright (d) 2020-2021 J. F. Lucas.  All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,14 +30,13 @@ DEST=${DEST:-/opt/plugins}
 # /path/to/plugin/wagon|/path/to/type/file
 PLUGINS=\
 "\
-/dcaepolicyplugin/2.4.0/dcaepolicyplugin-2.4.0-py27-none-linux_x86_64.wgn|/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
-/relationshipplugin/1.1.0/relationshipplugin-1.1.0-py27-none-linux_x86_64.wgn|/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
-/k8splugin/3.5.1/k8splugin-3.5.1-py27-none-linux_x86_64.wgn|/k8splugin/3.5.1/k8splugin_types.yaml \
-/clamppolicyplugin/1.1.0/clamppolicyplugin-1.1.0-py27-none-linux_x86_64.wgn|/clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
-/dmaap/1.5.0/dmaap-1.5.0-py27-none-linux_x86_64.wgn|/dmaap/1.5.0/dmaap_types.yaml \
-/helm/4.2.0/helm-4.2.0-py27-none-linux_x86_64.wgn|/helm/4.2.0/helm_types.yaml \
-/pgaas/1.3.0/pgaas-1.3.0-py27-none-linux_x86_64.wgn|/pgaas/1.3.0/pgaas_types.yaml \
-/sshkeyshare/1.2.0/sshkeyshare-1.2.0-py27-none-linux_x86_64.wgn|/sshkeyshare/1.2.0/sshkeyshare_types.yaml
+/dcaepolicyplugin/2.4.0/dcaepolicyplugin-2.4.0-py36-none-linux_x86_64.wgn|/dcaepolicyplugin/2.4.0/dcaepolicyplugin_types.yaml \
+/relationshipplugin/1.1.0/relationshipplugin-1.1.0-py36-none-linux_x86_64.wgn|/relationshipplugin/1.1.0/relationshipplugin_types.yaml \
+/k8splugin/3.5.1/k8splugin-3.5.1-py36-none-linux_x86_64.wgn|/k8splugin/3.5.1/k8splugin_types.yaml \
+/clamppolicyplugin/1.1.0/clamppolicyplugin-1.1.0-py36-none-linux_x86_64.wgn|/clamppolicyplugin/1.1.0/clamppolicyplugin_types.yaml \
+/dmaap/1.5.0/dmaap-1.5.0-py36-none-linux_x86_64.wgn|/dmaap/1.5.0/dmaap_types.yaml \
+/pgaas/1.3.0/pgaas-1.3.0-py36-none-linux_x86_64.wgn|/pgaas/1.3.0/pgaas_types.yaml \
+/sshkeyshare/1.2.0/sshkeyshare-1.2.0-py36-none-linux_x86_64.wgn|/sshkeyshare/1.2.0/sshkeyshare_types.yaml
 "
 
 mkdir -p ${DEST}
diff --git a/cm-container/scripts/load-plugins.sh b/cm-container/scripts/load-plugins.sh
index c93f3d9..4ac3c6d 100755
--- a/cm-container/scripts/load-plugins.sh
+++ b/cm-container/scripts/load-plugins.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 # Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2020-2021 J. F. Lucas.  All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -77,7 +78,6 @@ function install_plugin {
 
 set -ex
 
-
 # Wait for Cloudify Manager to come up
 while ! /scripts/cloudify-ready.sh
 do
@@ -85,6 +85,9 @@ do
     sleep 15
 done
 
+# Set nullglob to handle the case of an empty plugin directory
+shopt -s nullglob
+
 if [[ ! -f ${PLUGINS_LOADED} ]]
 then
 
diff --git a/cm-container/scripts/set-admin-password.sh b/cm-container/scripts/set-admin-password.sh
index d8c4121..04bd375 100755
--- a/cm-container/scripts/set-admin-password.sh
+++ b/cm-container/scripts/set-admin-password.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 # ============LICENSE_START=======================================================
 # Copyright (c) 2020 AT&T Intellectual Property. All rights reserved.
+# Copyright (c) 2020-2021 J. F. Lucas.  All rights reserved.
 # ================================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -35,7 +36,7 @@ CMPASS=${CMPASS:-admin}
 
 echo "Set Cloudify's admin password"
 cd /opt/manager
-cfy_manager --reset_admin_password $CMPASS || ./env/bin/python reset_admin.py -p $CMPASS
+cfy_manager reset-admin-password $CMPASS 
 
 echo "Set the password used by the cfy client"
 cfy profile set -p $CMPASS