Use global storage templates for PVC

OOM has now templates in order to create the needed PVC, using:

* a PV with a specific class when using a common nfs mount path between
nodes (sames as today use) --> is the default behavior today
* or a storage class if we want to use dynamic PV.
  On this case, we use (in order of priority):
  - persistence.storageClassOverride if set on the chart
  - global.persistence.storageClass if set globally
  - persistence.storageClass if set on the chart

I've also created a PV/PVC for elasticsearch has it was hardcoded to
hostPath.

I've also removed hostPath for job logs (aai-traversal logs were not put
into the desired hostPath)

And I moved the 3 jobs of aai-graphadmin into 3 files as it's a better
practice (several object in the same file can lead to unexpected
behavior).

Behavior is supposed to be exactly the same as before.

Issue-ID: OOM-1227
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: I71c6cee8d7c33ef7bd39c40a43c26b377d4ceec3

5 files changed, 293 insertions, 131 deletions

diff --git a/components/aai-graphadmin/templates/job copy-db-backup.yaml b/components/aai-graphadmin/templates/job copy-db-backup.yaml
new file mode 100644
index 0000000..80ccca3
--- /dev/null
+++ b/components/aai-graphadmin/templates/job copy-db-backup.yaml
@@ -0,0 +1,140 @@
+#
+# ============LICENSE_START=======================================================
+# org.onap.aai
+# ================================================================================
+# Copyright © 2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# In ONAP, the following job will always be run on each installation
+# The following job will go through the latest oxm and
+# create properties based on the data type defined in the oxm
+# and create the required indexes for the appropriate properties
+# This can be run multiple times as the code if the index or property already exists
+# then the index or property won't be created again
+# NOTE - During the execution of the createSchema job, there should
+# be no other janusgraph connection to the graph as its the reason
+# that resources traversal and graphadmin wait until this job is done
+# If you are using an existing cassandra cluster not coming from oom
+# then it is your job to ensure that there are no connections to the database
+
+{{- if .Values.global.jobs.migration.enabled }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "common.fullname" . }}-db-backup
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}-job
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+  annotations:
+    "helm.sh/hook": pre-upgrade,pre-install
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  backoffLimit: 20
+  template:
+    metadata:
+      labels:
+        app: {{ include "common.name" . }}-job
+        release: {{ .Release.Name }}
+      name: {{ include "common.name" . }}
+    spec:
+      {{ if eq .Values.global.jobs.migration.remoteCassandra.enabled false }}
+      initContainers:
+      - command:
+        - /bin/bash
+        - -c
+        - /root/ready.py --container-name aai-cassandra --timeout 1 || /root/ready.py --container-name cassandra
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        name: {{ include "common.name" . }}-db-backup-readiness
+      {{- end }}
+      containers:
+      - name: {{ include "common.name" . }}-db-backup-job
+        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        command:
+        - /bin/bash
+        - docker-entrypoint.sh
+        - dataSnapshot.sh
+        env:
+        - name: LOCAL_USER_ID
+          value: {{ .Values.global.config.userId | quote }}
+        - name: LOCAL_GROUP_ID
+          value: {{ .Values.global.config.groupId | quote }}
+        volumeMounts:
+        - mountPath: /etc/localtime
+          name: localtime
+          readOnly: true
+        - mountPath: /opt/app/aai-graphadmin/logs/data/dataSnapshots
+          name: {{ include "common.fullname" . }}-snapshots
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-realtime.properties
+          name: {{ include "common.fullname" . }}-migration
+          subPath: janusgraph-migration-real.properties
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-cached.properties
+          name: {{ include "common.fullname" . }}-migration
+          subPath: janusgraph-migration-cached.properties
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/aaiconfig.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: aaiconfig.properties
+        - mountPath: /opt/aai/logroot/AAI-RES/
+          name: {{ include "common.fullname" . }}-logs
+        - mountPath: /opt/app/aai-graphadmin/resources/logback.xml
+          name: {{ include "common.fullname" . }}-config
+          subPath: logback.xml
+        - mountPath: /opt/app/aai-graphadmin/resources/localhost-access-logback.xml
+          name: {{ include "common.fullname" . }}-config
+          subPath: localhost-access-logback.xml
+        - mountPath: /opt/app/aai-graphadmin/resources/application.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: application.properties
+        resources:
+{{ include "common.resources" . | indent 10 }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+      {{- end -}}
+      {{- if .Values.affinity }}
+      affinity:
+{{ toYaml .Values.affinity | indent 8 }}
+      {{- end }}
+      volumes:
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      - name: {{ include "common.fullname" . }}-logs
+        emptyDir: {}
+      - name: {{ include "common.fullname" . }}-config
+        configMap:
+         name: {{ include "common.fullname" . }}-configmap
+      - name: {{ include "common.fullname" . }}-migration
+        configMap:
+         name: {{ include "common.fullname" . }}-migration-configmap
+      - name: {{ include "common.fullname" . }}-snapshots
+        persistentVolumeClaim:
+          claimName: {{ include "common.fullname" . }}-migration
+      restartPolicy: Never
+      imagePullSecrets:
+      - name: "{{ include "common.namespace" . }}-docker-registry-key"
+{{ end }}
diff --git a/components/aai-graphadmin/templates/job-create-db.yaml b/components/aai-graphadmin/templates/job-create-db.yaml
new file mode 100644
index 0000000..1647260
--- /dev/null
+++ b/components/aai-graphadmin/templates/job-create-db.yaml
@@ -0,0 +1,149 @@
+#
+# ============LICENSE_START=======================================================
+# org.onap.aai
+# ================================================================================
+# Copyright © 2018 AT&T Intellectual Property. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+
+# In ONAP, the following job will always be run on each installation
+# The following job will go through the latest oxm and
+# create properties based on the data type defined in the oxm
+# and create the required indexes for the appropriate properties
+# This can be run multiple times as the code if the index or property already exists
+# then the index or property won't be created again
+# NOTE - During the execution of the createSchema job, there should
+# be no other janusgraph connection to the graph as its the reason
+# that resources traversal and graphadmin wait until this job is done
+# If you are using an existing cassandra cluster not coming from oom
+# then it is your job to ensure that there are no connections to the database
+
+{{- if and ( not .Values.global.jobs.migration.enabled ) ( .Values.global.jobs.createSchema.enabled ) }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "common.fullname" . }}-create-db-schema
+  namespace: {{ include "common.namespace" . }}
+  labels:
+    app: {{ include "common.name" . }}-job
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  backoffLimit: 20
+  template:
+    metadata:
+      labels:
+        app: {{ include "common.name" . }}-job
+        release: {{ .Release.Name }}
+      name: {{ include "common.name" . }}
+    spec:
+      initContainers:
+      - command:
+        - /root/ready.py
+        args:
+        - --container-name
+        {{- if .Values.global.cassandra.localCluster }}
+        - aai-cassandra
+        {{- else }}
+        - cassandra
+        {{- end }}
+        - --container-name
+        - aai-schema-service
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        name: {{ include "common.name" . }}-readiness
+      containers:
+      - name: {{ include "common.name" . }}-job
+        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        command:
+        - /bin/bash
+        - docker-entrypoint.sh
+        - createDBSchema.sh
+        env:
+        - name: LOCAL_USER_ID
+          value: {{ .Values.global.config.userId | quote }}
+        - name: LOCAL_GROUP_ID
+          value: {{ .Values.global.config.groupId | quote }}
+        volumeMounts:
+        - mountPath: /etc/localtime
+          name: localtime
+          readOnly: true
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-realtime.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: janusgraph-realtime.properties
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-cached.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: janusgraph-cached.properties
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/aaiconfig.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: aaiconfig.properties
+        - mountPath: /opt/aai/logroot/AAI-GA
+          name: {{ include "common.fullname" . }}-logs
+        - mountPath: /opt/app/aai-graphadmin/resources/logback.xml
+          name: {{ include "common.fullname" . }}-config
+          subPath: logback.xml
+        - mountPath: /opt/app/aai-graphadmin/resources/localhost-access-logback.xml
+          name: {{ include "common.fullname" . }}-config
+          subPath: localhost-access-logback.xml
+        - mountPath: /opt/app/aai-graphadmin/resources/application.properties
+          name: {{ include "common.fullname" . }}-config
+          subPath: application.properties
+          {{ $global := . }}
+          {{ range $job := .Values.global.config.auth.files }}
+        - mountPath: /opt/app/aai-graphadmin/resources/etc/auth/{{ . }}
+          name: {{ include "common.fullname" $global }}-auth-truststore-sec
+          subPath: {{ . }}
+          {{ end }}
+        resources:
+{{ include "common.resources" . }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+      {{- end -}}
+      {{- if .Values.affinity }}
+      affinity:
+{{ toYaml .Values.affinity | indent 8 }}
+      {{- end }}
+      volumes:
+      - name: localtime
+        hostPath:
+          path: /etc/localtime
+      - name: filebeat-conf
+        configMap:
+          name: aai-filebeat
+      - name: {{ include "common.fullname" . }}-logs
+        emptyDir: {}
+      - name: {{ include "common.fullname" . }}-config
+        configMap:
+         name: {{ include "common.fullname" . }}-configmap
+      - name: {{ include "common.fullname" . }}-auth-truststore-sec
+        secret:
+         secretName: aai-common-truststore
+         items:
+          {{ range $job := .Values.global.config.auth.files }}
+           - key: {{ . }}
+             path: {{ . }}
+          {{ end }}
+      restartPolicy: Never
+      imagePullSecrets:
+      - name: "{{ include "common.namespace" . }}-docker-registry-key"
+{{ end }}
\ No newline at end of file
diff --git a/components/aai-graphadmin/templates/job.yaml b/components/aai-graphadmin/templates/job-migration.yaml
index e5b673b..9815e66 100644
--- a/components/aai-graphadmin/templates/job.yaml
+++ b/components/aai-graphadmin/templates/job-migration.yaml
@@ -29,125 +29,6 @@
 # If you are using an existing cassandra cluster not coming from oom
 # then it is your job to ensure that there are no connections to the database
 
-{{- if and ( not .Values.global.jobs.migration.enabled ) ( .Values.global.jobs.createSchema.enabled ) }}
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: {{ include "common.fullname" . }}-create-db-schema
-  namespace: {{ include "common.namespace" . }}
-  labels:
-    app: {{ include "common.name" . }}-job
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
-spec:
-  backoffLimit: 20
-  template:
-    metadata:
-      labels:
-        app: {{ include "common.name" . }}-job
-        release: {{ .Release.Name }}
-      name: {{ include "common.name" . }}
-    spec:
-      initContainers:
-      - command:
-        - /root/ready.py
-        args:
-        - --container-name
-        {{- if .Values.global.cassandra.localCluster }}
-        - aai-cassandra
-        {{- else }}
-        - cassandra
-        {{- end }}
-        - --container-name
-        - aai-schema-service
-        env:
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.namespace
-        image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
-        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-        name: {{ include "common.name" . }}-readiness
-      containers:
-      - name: {{ include "common.name" . }}-job
-        image: "{{ include "common.repository" . }}/{{ .Values.image }}"
-        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-        command:
-        - /bin/bash
-        - docker-entrypoint.sh
-        - createDBSchema.sh
-        env:
-        - name: LOCAL_USER_ID
-          value: {{ .Values.global.config.userId | quote }}
-        - name: LOCAL_GROUP_ID
-          value: {{ .Values.global.config.groupId | quote }}
-        volumeMounts:
-        - mountPath: /etc/localtime
-          name: localtime
-          readOnly: true
-        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-realtime.properties
-          name: {{ include "common.fullname" . }}-config
-          subPath: janusgraph-realtime.properties
-        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/janusgraph-cached.properties
-          name: {{ include "common.fullname" . }}-config
-          subPath: janusgraph-cached.properties
-        - mountPath: /opt/app/aai-graphadmin/resources/etc/appprops/aaiconfig.properties
-          name: {{ include "common.fullname" . }}-config
-          subPath: aaiconfig.properties
-        - mountPath: /opt/aai/logroot/AAI-GA
-          name: {{ include "common.fullname" . }}-logs
-        - mountPath: /opt/app/aai-graphadmin/resources/logback.xml
-          name: {{ include "common.fullname" . }}-config
-          subPath: logback.xml
-        - mountPath: /opt/app/aai-graphadmin/resources/localhost-access-logback.xml
-          name: {{ include "common.fullname" . }}-config
-          subPath: localhost-access-logback.xml
-        - mountPath: /opt/app/aai-graphadmin/resources/application.properties
-          name: {{ include "common.fullname" . }}-config
-          subPath: application.properties
-          {{ $global := . }}
-          {{ range $job := .Values.global.config.auth.files }}
-        - mountPath: /opt/app/aai-graphadmin/resources/etc/auth/{{ . }}
-          name: {{ include "common.fullname" $global }}-auth-truststore-sec
-          subPath: {{ . }}
-          {{ end }}
-        resources:
-{{ include "common.resources" . }}
-      {{- if .Values.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.nodeSelector | indent 8 }}
-      {{- end -}}
-      {{- if .Values.affinity }}
-      affinity:
-{{ toYaml .Values.affinity | indent 8 }}
-      {{- end }}
-      volumes:
-      - name: localtime
-        hostPath:
-          path: /etc/localtime
-      - name: filebeat-conf
-        configMap:
-          name: aai-filebeat
-      - name: {{ include "common.fullname" . }}-logs
-        hostPath:
-          path: {{ .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath }}-create-db-schema
-      - name: {{ include "common.fullname" . }}-config
-        configMap:
-         name: {{ include "common.fullname" . }}-configmap
-      - name: {{ include "common.fullname" . }}-auth-truststore-sec
-        secret:
-         secretName: aai-common-truststore
-         items:
-          {{ range $job := .Values.global.config.auth.files }}
-           - key: {{ . }}
-             path: {{ . }}
-          {{ end }}
-      restartPolicy: Never
-      imagePullSecrets:
-      - name: "{{ include "common.namespace" . }}-docker-registry-key"
-{{ end }}
 {{- if .Values.global.jobs.migration.enabled }}
 ---
 apiVersion: batch/v1
@@ -425,4 +306,3 @@ spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
 {{ end }}
-
diff --git a/components/aai-graphadmin/templates/pv.yaml b/components/aai-graphadmin/templates/pv.yaml
index f678331..e790e6a 100644
--- a/components/aai-graphadmin/templates/pv.yaml
+++ b/components/aai-graphadmin/templates/pv.yaml
@@ -15,6 +15,7 @@
 */}}
 
 {{- if .Values.global.jobs.migration.enabled -}}
+{{- if eq "True" (include "common.needPV" .) -}}
 kind: PersistentVolume
 apiVersion: v1
 metadata:
@@ -36,7 +37,8 @@ spec:
   accessModes:
     - {{ .Values.persistence.accessMode }}
   persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }}
+  storageClassName: "{{ include "common.fullname" . }}-data"
   hostPath:
     path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ .Release.Name }}/{{ .Values.persistence.mountSubPath1 }}
 {{- end -}}
-
+{{- end -}}
diff --git a/components/aai-graphadmin/templates/pvc.yaml b/components/aai-graphadmin/templates/pvc.yaml
index 582afe9..14e07d1 100644
--- a/components/aai-graphadmin/templates/pvc.yaml
+++ b/components/aai-graphadmin/templates/pvc.yaml
@@ -33,19 +33,10 @@ metadata:
 {{ toYaml .Values.persistence.annotations | indent 4 }}
 {{- end }}
 spec:
-  selector:
-    matchLabels:
-      name: {{ include "common.fullname" . }}
   accessModes:
     - {{ .Values.persistence.accessMode }}
   resources:
     requests:
       storage: {{ .Values.persistence.size }}
-{{- if .Values.persistence.storageClass }}
-{{- if (eq "-" .Values.persistence.storageClass) }}
-  storageClassName: ""
-{{- else }}
-  storageClassName: "{{ .Values.persistence.storageClass }}"
-{{- end }}
-{{- end }}
+  storageClassName: {{ include "common.storageClass" . }}
 {{- end -}}