Helm and blueprint implementation of datalake

1. Deployment Helm chart.
2. helm plugin based cloudify blueprints.
3. fix pom.xml and directory location for CI.

Issue-ID: DCAEGEN2-2008
Signed-off-by: Kate Hsuan <kate.hsuan@qct.io>
Change-Id: If24524576e72392b66ff556237154922b213a29b

8 files changed, 406 insertions, 0 deletions

diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/NOTES.txt b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/NOTES.txt
new file mode 100644
index 00000000..4e97715e
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/NOTES.txt
@@ -0,0 +1,39 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "dcae-datalake-feeder.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "dcae-datalake-feeder.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "dcae-datalake-feeder.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "dcae-datalake-feeder.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl port-forward $POD_NAME 8080:80
+{{- end }}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/_helpers.tpl b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/_helpers.tpl
new file mode 100644
index 00000000..272853ea
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "dcae-datalake-feeder.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "dcae-datalake-feeder.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "dcae-datalake-feeder.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "dcae-datalake-feeder.labels" -}}
+app.kubernetes.io/name: {{ include "dcae-datalake-feeder.name" . }}
+helm.sh/chart: {{ include "dcae-datalake-feeder.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/configmap.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/configmap.yaml
new file mode 100644
index 00000000..f7aca237
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/configmap.yaml
@@ -0,0 +1,27 @@
+#   Copyright (C) 2020 QCT All rights reserved.
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  ============LICENSE_END=========================================================
+#  @author:kate.hsuan@qct.io
+apiVersion: v1
+
+kind: ConfigMap
+metadata:
+  name: {{ include "dcae-datalake-feeder.fullname" . }}-configmap
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+data:
+{{ tpl (.Files.Glob "resources/config/overrides/*").AsConfig . | indent 2 }}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/deployment.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/deployment.yaml
new file mode 100644
index 00000000..4c9dffcd
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/deployment.yaml
@@ -0,0 +1,96 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "dcae-datalake-feeder.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+spec:
+  hostname: dcae-datalake-feeder-feeder
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "dcae-datalake-feeder.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "dcae-datalake-feeder.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+    {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+      initContainers:
+      - command:
+        - /root/job_complete.py
+        args:
+        - -j
+        - {{ .Release.Name }}-dcae-datalake-feeder-dbinit-job
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+        name: datalake-readiness
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+          - name: HOSTNAME
+            value: k8s-datalake
+          - name: CONSUL_HOST
+            value: consul
+          - name: CONFIG_BINDING_SERVICE
+            value: config-binding-service      
+          ports:
+            - name: http
+              containerPort: 1680
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/datalake
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ .Release.Name }}-dcae-datalake-feeder-configmap
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/ingress.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/ingress.yaml
new file mode 100644
index 00000000..af724c12
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/ingress.yaml
@@ -0,0 +1,53 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "dcae-datalake-feeder.fullname" . -}}
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.ingress.tls }}
+  tls:
+  {{- range .Values.ingress.tls }}
+    - hosts:
+      {{- range .hosts }}
+        - {{ . | quote }}
+      {{- end }}
+      secretName: {{ .secretName }}
+  {{- end }}
+{{- end }}
+  rules:
+  {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+        {{- range .paths }}
+          - path: {{ . }}
+            backend:
+              serviceName: {{ $fullName }}
+              servicePort: http
+        {{- end }}
+  {{- end }}
+{{- end }}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/job.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/job.yaml
new file mode 100644
index 00000000..a990fb75
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/job.yaml
@@ -0,0 +1,57 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "dcae-datalake-feeder.fullname" . }}-dbinit-job
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+spec:
+  template:
+    spec:
+      #initContainers:
+      #- name: datalake-readiness
+        #command:
+        #- /root/ready.py
+        #args:
+        #- --container-name
+        #- {{ .Values.config.mariadbGalera.chartName }}
+        #env:
+        #- name: NAMESPACE
+        #  valueFrom:
+        #    fieldRef:
+        #      apiVersion: v1
+        #      fieldPath: metadata.namespace
+        #image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+        #imagePullPolicy: Always
+
+      containers:
+      - name: datalake-db-init
+        image: nexus3.onap.org:10001/onap/org.onap.dcaegen2.services.datalake.dbinit:1.0.0
+        env:
+        - name: MYSQL_HOST
+          value: "{{ .Values.config.mariadbGalera.serviceName }}"
+        - name: MYSQL_PORT
+          value: "3306"
+        - name: MYSQL_ROOT_PASSWORD
+          value: "secretpassword"
+        - name: PRESTO_HOST
+          value: "{{ .Values.config.presto.serviceName }}"
+        command: ["sh",  "run.sh"]
+      restartPolicy: Never
+  backoffLimit: 4
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/service.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/service.yaml
new file mode 100644
index 00000000..918f0bc8
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/service.yaml
@@ -0,0 +1,35 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: dl-feeder-api
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+spec:
+  type: NodePort
+  ports:
+    - port: 1680
+      targetPort: 1680
+      nodePort: {{ .Values.global.nodePortPrefixExt }}{{ .Values.service.nodePort }}
+      protocol: TCP
+      name: datalake-api
+  selector:
+    app.kubernetes.io/name: {{ include "dcae-datalake-feeder.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
diff --git a/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/serviceCusterIP.yaml b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/serviceCusterIP.yaml
new file mode 100644
index 00000000..32043b43
--- /dev/null
+++ b/components/datalake-handler/dpo/helm_chart/dcae-datalake/charts/dcae-datalake-feeder/templates/serviceCusterIP.yaml
@@ -0,0 +1,36 @@
+# ============LICENSE_START==========================================
+# ===================================================================
+#  Copyright (c) 2020 QCT
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#============LICENSE_END============================================
+
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: dl-feeder
+  labels:
+{{ include "dcae-datalake-feeder.labels" . | indent 4 }}
+spec:
+  clusterIP: None
+  type: ClusterIP
+  ports:
+  - name: datalake-api-cluster
+    port: 1680
+    protocol: TCP
+    targetPort: 1680
+  selector:
+    app.kubernetes.io/name: {{ include "dcae-datalake-feeder.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+