diff options
author | ChrisC <christophe.closset@intl.att.com> | 2020-09-11 18:39:23 +0200 |
---|---|---|
committer | ChrisC <christophe.closset@intl.att.com> | 2020-09-17 15:15:13 +0200 |
commit | 2325efd0b6f8b094f6a801bf55d6ff6f53e9cbfa (patch) | |
tree | ca6323d4e590c648b333cfc87122018d3e273323 /kubernetes/clamp/components/clamp-dash-logstash | |
parent | 98efeea41f5617760fcc5fdb6718409b69684db9 (diff) |
[CLAMP] AAF certificate using certinializer
use of auto-generated certificates via AAF side-car
at OOM deployment time for CLAMP.
Issue-ID: CLAMP-884
Change-Id: I24f5a119714a5e46c4d0c152c03b6bc545135b8e
Signed-off-by: osgn422w <gervais-martial.ngueko@intl.att.com>
Signed-off-by: ChrisC <christophe.closset@intl.att.com>
Diffstat (limited to 'kubernetes/clamp/components/clamp-dash-logstash')
8 files changed, 630 insertions, 0 deletions
diff --git a/kubernetes/clamp/components/clamp-dash-logstash/Chart.yaml b/kubernetes/clamp/components/clamp-dash-logstash/Chart.yaml new file mode 100644 index 0000000000..9fc0317fd3 --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/Chart.yaml @@ -0,0 +1,19 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: ONAP Clamp Dashboard Logstash +name: clamp-dash-logstash +version: 6.0.0 diff --git a/kubernetes/clamp/components/clamp-dash-logstash/requirements.yaml b/kubernetes/clamp/components/clamp-dash-logstash/requirements.yaml new file mode 100644 index 0000000000..caff1e5dc4 --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/requirements.yaml @@ -0,0 +1,19 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~6.x-0 + repository: '@local' diff --git a/kubernetes/clamp/components/clamp-dash-logstash/resources/config/logstash.yml b/kubernetes/clamp/components/clamp-dash-logstash/resources/config/logstash.yml new file mode 100644 index 0000000000..cecd5b18c8 --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/resources/config/logstash.yml @@ -0,0 +1,24 @@ +# Copyright © 2020 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +http.host: "0.0.0.0" +## Path where pipeline configurations reside +path.config: /usr/share/logstash/pipeline + +## Type of queue : memeory based or file based +#queue.type: persisted +## Size of queue +#queue.max_bytes: 1024mb +## Setting true makes logstash check periodically for change in pipeline configurations +config.reload.automatic: true + diff --git a/kubernetes/clamp/components/clamp-dash-logstash/resources/config/pipeline.conf b/kubernetes/clamp/components/clamp-dash-logstash/resources/config/pipeline.conf new file mode 100644 index 0000000000..c005fcca3e --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/resources/config/pipeline.conf @@ -0,0 +1,256 @@ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +input { + http_poller { + urls => { + event_queue => { + method => get + url => "${dmaap_base_url}/events/${event_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + topic => "${event_topic}" + tags => [ "dmaap_source" ] + } + notification_queue => { + method => get + url => "${dmaap_base_url}/events/${notification_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + topic => "${notification_topic}" + tags => [ "dmaap_source" ] + } + request_queue => { + method => get + url => "${dmaap_base_url}/events/${request_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + topic => "${request_topic}" + tags => [ "dmaap_source" ] + } + } + socket_timeout => 30 + request_timeout => 30 + schedule => { "every" => "1m" } + codec => "plain" + cacert => "/certs.d/aafca.pem" + } +} + + +filter { + # avoid noise if no entry in the list + if [message] == "[]" { + drop { } + } + + if [http_request_failure] or [@metadata][code] != 200 { + mutate { + add_tag => [ "error" ] + } + } + + if "dmaap_source" in [@metadata][request][tags] { + # + # Dmaap provides a json list, whose items are Strings containing the event + # provided to Dmaap, which itself is an escaped json. + # + # We first need to parse the json as we have to use the plaintext as it cannot + # work with list of events, then split that list into multiple string events, + # that we then transform into json. + # + json { + source => "[message]" + target => "message" + } + + split { + field => "message" + } + json { + source => "message" + } + mutate { + remove_field => [ "message" ] + } + } + + # + # Some timestamps are expressed as milliseconds, some are in microseconds + # + if [closedLoopAlarmStart] { + ruby { + code => " + if event.get('closedLoopAlarmStart').to_s.to_i(10) > 9999999999999 + event.set('closedLoopAlarmStart', event.get('closedLoopAlarmStart').to_s.to_i(10) / 1000) + else + event.set('closedLoopAlarmStart', event.get('closedLoopAlarmStart').to_s.to_i(10)) + end + " + } + date { + match => [ "closedLoopAlarmStart", UNIX_MS ] + target => "closedLoopAlarmStart" + } + } + + if [closedLoopAlarmEnd] { + ruby { + code => " + if event.get('closedLoopAlarmEnd').to_s.to_i(10) > 9999999999999 + event.set('closedLoopAlarmEnd', event.get('closedLoopAlarmEnd').to_s.to_i(10) / 1000) + else + event.set('closedLoopAlarmEnd', event.get('closedLoopAlarmEnd').to_s.to_i(10)) + end + " + } + date { + match => [ "closedLoopAlarmEnd", UNIX_MS ] + target => "closedLoopAlarmEnd" + } + + } + + + # + # Notification time are expressed under the form "yyyy-MM-dd HH:mm:ss", which + # is close to ISO8601, but lacks of T as spacer: "yyyy-MM-ddTHH:mm:ss" + # + if [notificationTime] { + mutate { + gsub => [ + "notificationTime", " ", "T" + ] + } + date { + match => [ "notificationTime", ISO8601 ] + target => "notificationTime" + } + } + + + # + # Renaming some fields for readability + # + if [AAI][generic-vnf.vnf-name] { + mutate { + add_field => { "vnfName" => "%{[AAI][generic-vnf.vnf-name]}" } + } + } + if [AAI][generic-vnf.vnf-type] { + mutate { + add_field => { "vnfType" => "%{[AAI][generic-vnf.vnf-type]}" } + } + } + if [AAI][vserver.vserver-name] { + mutate { + add_field => { "vmName" => "%{[AAI][vserver.vserver-name]}" } + } + } + if [AAI][complex.city] { + mutate { + add_field => { "locationCity" => "%{[AAI][complex.city]}" } + } + } + if [AAI][complex.state] { + mutate { + add_field => { "locationState" => "%{[AAI][complex.state]}" } + } + } + + + # + # Adding some flags to ease aggregation + # + if [closedLoopEventStatus] =~ /(?i)ABATED/ { + mutate { + add_field => { "flagAbated" => "1" } + } + } + if [notification] =~ /^.*?(?:\b|_)FINAL(?:\b|_).*?(?:\b|_)FAILURE(?:\b|_).*?$/ { + mutate { + add_field => { "flagFinalFailure" => "1" } + } + } + + + if "error" not in [@metadata][request][tags]{ + # + # Creating data for a secondary index + # + clone { + clones => [ "event-cl-aggs" ] + add_tag => [ "event-cl-aggs" ] + } + + if "event-cl-aggs" in [@metadata][request][tags]{ + # + # we only need a few fields for aggregations; remove all fields from clone except : + # vmName,vnfName,vnfType,requestID,closedLoopAlarmStart, closedLoopControlName,closedLoopAlarmEnd,abated,nbrDmaapevents,finalFailure + # + prune { + whitelist_names => ["^@.*$","^topic$","^type$","^tags$","^flagFinalFailure$","^flagAbated$","^locationState$","^locationCity$","^vmName$","^vnfName$","^vnfType$","^requestID$","^closedLoopAlarmStart$","^closedLoopControlName$","^closedLoopAlarmEnd$","^target$","^target_type$","^triggerSourceName$","^policyScope$","^policyName$","^policyVersion$"] + } + + } + } +} + + +output { + stdout { + codec => rubydebug + } + + if "error" in [tags] { + elasticsearch { + codec => "json" + cacert => "/clamp-cert/ca-certs.pem" + ssl_certificate_verification => false + hosts => ["${elasticsearch_base_url}"] + user => ["${logstash_user}"] + password => ["${logstash_pwd}"] + index => "errors-%{+YYYY.MM.DD}" + doc_as_upsert => true + } + + } else if "event-cl-aggs" in [tags] { + elasticsearch { + codec => "json" + hosts => ["${elasticsearch_base_url}"] + cacert => "/clamp-cert/ca-certs.pem" + ssl_certificate_verification => false + user => ["${logstash_user}"] + password => ["${logstash_pwd}"] + document_id => "%{requestID}" + index => "events-cl-%{+YYYY.MM.DD}" # creates daily indexes for control loop + doc_as_upsert => true + action => "update" + } + + } else { + elasticsearch { + codec => "json" + hosts => ["${elasticsearch_base_url}"] + cacert => "/clamp-cert/ca-certs.pem" + ssl_certificate_verification => false + user => ["${logstash_user}"] + password => ["${logstash_pwd}"] + index => "events-%{+YYYY.MM.DD}" # creates daily indexes + doc_as_upsert => true + } + } +} diff --git a/kubernetes/clamp/components/clamp-dash-logstash/templates/configmap.yaml b/kubernetes/clamp/components/clamp-dash-logstash/templates/configmap.yaml new file mode 100644 index 0000000000..4278a6e6d3 --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/templates/configmap.yaml @@ -0,0 +1,27 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/clamp/components/clamp-dash-logstash/templates/deployment.yaml b/kubernetes/clamp/components/clamp-dash-logstash/templates/deployment.yaml new file mode 100644 index 0000000000..acd108d2cf --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/templates/deployment.yaml @@ -0,0 +1,127 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ include "common.name" . }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} + spec: + initContainers: + - command: + - /app/ready.py + args: + - --container-name + - clamp-dash-es + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: "{{ include "common.repository" . }}/{{ .Values.global.readinessImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-readiness + containers: + - name: {{ include "common.name" . }} + image: "{{ include "common.repository" . }}/{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + env: + - name: dmaap_consumer_group + value: "{{ .Values.config.dmaapConsumerGroup }}" + - name: dmaap_consumer_id + value: "{{ .Values.config.dmaapConsumerId }}" + - name: event_topic + value: "{{ .Values.config.eventTopic }}" + - name: notification_topic + value: "{{ .Values.config.notificationTopic }}" + - name: request_topic + value: "{{ .Values.config.requestTopic }}" + - name: dmaap_base_url + value: {{ ternary "https" "http" .Values.security.ssl.enabled }}://{{ .Values.config.dmaapHost }}.{{ include "common.namespace" . }}:{{ .Values.config.dmaapPort }} + - name: logstash_user + value: "{{ .Values.config.logstash_user }}" + - name: logstash_pwd + value: "{{ .Values.config.logstash_pwd }}" + - name: elasticsearch_base_url + value: "{{ ternary "https" "http" .Values.security.ssl.enabled }}://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.config.elasticsearchPort}}" + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ include "common.servicename" . }} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} +# disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + {{ end -}} + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /usr/share/logstash/config/logstash.yml + name: {{ include "common.fullname" . }} + subPath: logstash.yml + - mountPath: /usr/share/logstash/pipeline/logstash.conf + name: {{ include "common.fullname" . }} + subPath: pipeline.conf + resources: +{{ include "common.resources" . | indent 12 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 10 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 10 }} + {{- end }} + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }} + configMap: + name: {{ include "common.fullname" . }} + items: + - key: logstash.yml + path: logstash.yml + - key: pipeline.conf + path: pipeline.conf + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/clamp/components/clamp-dash-logstash/templates/service.yaml b/kubernetes/clamp/components/clamp-dash-logstash/templates/service.yaml new file mode 100644 index 0000000000..07d4a8f8ea --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/templates/service.yaml @@ -0,0 +1,41 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.servicename" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.externalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.config.portName }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.config.portName }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} diff --git a/kubernetes/clamp/components/clamp-dash-logstash/values.yaml b/kubernetes/clamp/components/clamp-dash-logstash/values.yaml new file mode 100644 index 0000000000..c2a522b1c8 --- /dev/null +++ b/kubernetes/clamp/components/clamp-dash-logstash/values.yaml @@ -0,0 +1,117 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + repositorySecret: eyJuZXh1czMub25hcC5vcmc6MTAwMDEiOnsidXNlcm5hbWUiOiJkb2NrZXIiLCJwYXNzd29yZCI6ImRvY2tlciIsImVtYWlsIjoiQCIsImF1dGgiOiJaRzlqYTJWeU9tUnZZMnRsY2c9PSJ9fQ== + readinessImage: onap/oom/readiness:3.0.1 + persistence: {} +flavor: small + +################################################################# +# Application configuration defaults. +################################################################# + +# application image +repository: nexus3.onap.org:10001 +image: onap/clamp-dashboard-logstash:5.0.3 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +config: + elasticsearchServiceName: cdash-es + elasticsearchPort: 9200 + dmaapHost: message-router + dmaapSchemeSSL: https + dmaapSchemeNoSSL: http + dmaapPort: 3905 + dmaapConsumerGroup: "clampdashboard" + dmaapConsumerId: "clampdashboard" + eventTopic: "DCAE-CL-EVENT" + notificationTopic: "POLICY-CL-MGT" + requestTopic: "APPC-CL" + logstash_user: "logstash" + logstash_pwd: "logstash" + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 900 + periodSeconds: 20 + timeoutSeconds: 5 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 900 + periodSeconds: 20 + timeoutSeconds: 5 + +security: + ssl: + enabled: true + +service: + #Example service definition with external, internal and node ports. + #Services may use any combination of ports depending on the 'type' of + #service being defined. + type: ClusterIP + name: cdash-ls + portName: cdash-ls-healthcheck + externalPort: 9600 + internalPort: 9600 +ingress: + enabled: false + +#resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # Example: + # Configure resource requests and limits + # ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # Minimum memory for development is 2 CPU cores and 4GB memory + # Minimum memory for production is 4 CPU cores and 8GB memory +resources: + small: + limits: + cpu: 1 + memory: 1.3Gi + requests: + cpu: 10m + memory: 750Mi + large: + limits: + cpu: 1 + memory: 1.3Gi + requests: + cpu: 10m + memory: 750Mi + unlimited: {} |