diff options
| -rw-r--r-- | cm-container/Dockerfile-template | 9 | ||||
| -rw-r--r-- | cm-container/README.md | 31 | ||||
| -rw-r--r-- | cm-container/pom.xml | 2 | ||||
| -rwxr-xr-x | cm-container/setup-secret.sh | 27 | ||||
| -rwxr-xr-x | cm-container/start-persistent.sh | 53 |
5 files changed, 119 insertions, 3 deletions
diff --git a/cm-container/Dockerfile-template b/cm-container/Dockerfile-template index 5c052a0..9462ff5 100644 --- a/cm-container/Dockerfile-template +++ b/cm-container/Dockerfile-template @@ -25,13 +25,16 @@ ENV CCSDK_REPO {{ ONAPTEMPLATE_RAWREPOURL_org_onap_ccsdk_platform_plugins_releas # Store type files locally RUN mkdir scripts -COPY get-type-files.sh dcae-cleanup.sh scripts/ +COPY get-type-files.sh dcae-cleanup.sh start-persistent.sh setup-secret.sh scripts/ # Load our type files and the Cloudify 3.4 type files +# Setup rc.local to set up k8s credentials for CM RUN scripts/get-type-files.sh ${TYPE_REPO} ${CCSDK_REPO}\ && mkdir /opt/manager/resources/spec/cloudify/3.4\ && curl -Ss https://cloudify.co/spec/cloudify/3.4/types.yaml > /opt/manager/resources/spec/cloudify/3.4/types.yaml\ && chown -R cfyuser:cfyuser /opt/manager/resources/spec/cloudify/3.4\ - && chmod +x scripts/*.sh + && chmod +x scripts/*.sh\ + && echo "/scripts/setup-secret.sh" >> /etc/rc.d/rc.local\ + && chmod +x /etc/rc.d/rc.local # Create mount point for CM config file RUN mkdir -p /opt/onap && chown cfyuser:cfyuser /opt/onap @@ -48,3 +51,5 @@ RUN pip install --upgrade pip==9.0.3 \ && virtualenv cfy42 \ && source cfy42/bin/activate \ && pip install cloudify==4.2 + +CMD ["/scripts/start-persistent.sh"] diff --git a/cm-container/README.md b/cm-container/README.md index a29423d..6e1e26e 100644 --- a/cm-container/README.md +++ b/cm-container/README.md @@ -31,3 +31,34 @@ In a Kubernetes environment, we expect that the <path_to_kubeconfile_file> and t We also expect that in a Kubernetes environment the external port mapping would not be needed. + +## Persistent Storage +In an ONAP deployment driven by OOM, Cloudify Manager will store data related to its state +in a Kubernetes PersistentVolume. If the Cloudify Manager pod is destroyed and recreated, +the new instance will have all of the state information from the previous run. + +To set up persistent, we replace the command run by the container (`CMD` in the Dockerfile) with +our own script `start-persistent.sh`. This script checks to see if a persistent volume has been +mounted in a well-known place (`/cfy-persist` in the container's file system). If so, the script +then checks to see if the persistent volume has been populated with data. There are two possibilities: +1. The persistent volume hasn't been populated, indicating that this is the first time Cloudify Manager is +being run in the current environment. In this case, the script copies state data from several directories in +the container file system into directories in the persistent volume. This is data (such as database schemas for +Cloudify Manager's internal postgres instance) that was generated when the original Cloudify Manager image was +created by Cloudify. +2. The persistent volume has been populated, indicating that this is not the first time Cloudify Manager is being +run in the current environment. The data in the persistent volume reflects the state that Cloudify Manager was in +when it exited at some point in the past. There's no need to copy data in this case. +In either case, the script will create symbolic links from the original data directories to the corresponding directories +in the persistent store. + +If there is no persistent volume mounted, the script does nothing to set up persistent data, and the container will have +no persistent storage. + +The last command in the script is the command from the original Cloudify version of the Cloudify Manager image. It runs `/sbin/init`, +which then brings up the many other processes needed for a working instance of Cloudify Manager. + +## The `setup-secret.sh` script +When Kubernetes starts a container, it mounts a directory containing the credentials that the container needs to access the Kubernetes API on the local Kubernetes cluster. The mountpoint is `/var/run/secrets/kubernetes.io/serviceaccount`. Something about the way that Cloudify Manager is started (possibly because `/sbin/init` is run) causes this mountpoint to be hidden. `setup-secret.sh` will recreated the directory if it's not present and symbolically link it to a copy of the credentials mounted at `/secret` in the container file system. This gives Cloudify Manager the credentials that the Kubernetes plugin needs to deploy Kubernetes-based DCAE components. + +`setup-secret.sh` needs to run after '/sbin/init'. The Dockerfile installs it in the `rc.local` script that runs at startup.
\ No newline at end of file diff --git a/cm-container/pom.xml b/cm-container/pom.xml index 895f3c3..5153799 100644 --- a/cm-container/pom.xml +++ b/cm-container/pom.xml @@ -27,7 +27,7 @@ limitations under the License. <groupId>org.onap.dcaegen2.deployments</groupId> <artifactId>cm-container</artifactId> <name>dcaegen2-deployments-cm-container</name> - <version>1.4.2</version> + <version>1.5.0</version> <url>http://maven.apache.org</url> <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> diff --git a/cm-container/setup-secret.sh b/cm-container/setup-secret.sh new file mode 100755 index 0000000..848ed28 --- /dev/null +++ b/cm-container/setup-secret.sh @@ -0,0 +1,27 @@ + +#!/bin/bash +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# Set up credentials for CM to access k8s +# (formerly done in a postStart hook, which now seems to have timing issues) + +set -ex + +if [ ! -d /var/run/secrets/kubernetes.io/serviceaccount ] +then + mkdir -p /var/run/secrets/kubernetes.io/ + ln -s /secret/ /var/run/secrets/kubernetes.io/serviceaccount +fi diff --git a/cm-container/start-persistent.sh b/cm-container/start-persistent.sh new file mode 100755 index 0000000..bf55da0 --- /dev/null +++ b/cm-container/start-persistent.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# Set up persistent storage for Cloudify Manager's state data + +PDIRS="/var/lib/pgsql/9.5/data /opt/manager/resources /opt/mgmtworker/env/plugins /opt/mgmtworker/work/deployments" +PSTORE="/cfy-persist" + +set -ex + +if [ -d "$PSTORE" ] +then + # the persistent mount point exists + if [ -z "$(ls -A $PSTORE)" ] + then + # there's nothing in the persistent store yet + # copy in the data from the container file system + for d in $PDIRS + do + p="$(dirname $d)" + mkdir -p "${PSTORE}$p" + cp -rp "$d" "${PSTORE}$p" + done + fi + # at this point, there is persistent storage possibly from a previous startup + # set up links from internal file system to persistent storage + for d in $PDIRS + do + if [ -d "$d" ] + then + mv $d $d-initial # move directory so we can create symlink + fi + ln -sf "$PSTORE/$d" "$(dirname $d)" + done +else + echo "No persistent storage available" +fi +# start up init, which brings up CM and supporting software +exec /sbin/init --log-target=journal 3>&1 + |