diff options
Diffstat (limited to 'kubernetes/clamp/charts/clamp-dash-logstash')
8 files changed, 448 insertions, 0 deletions
diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/Chart.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/Chart.yaml new file mode 100644 index 0000000000..d7a84183de --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/Chart.yaml @@ -0,0 +1,20 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +apiVersion: v1 +description: ONAP Clamp Dashboard Logstash +name: clamp-dash-logstash +version: 2.0.0 diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/requirements.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/requirements.yaml new file mode 100644 index 0000000000..2202d75b88 --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/requirements.yaml @@ -0,0 +1,20 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +dependencies: + - name: common + version: ~2.0.0 + repository: '@local'
\ No newline at end of file diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/logstash.yml b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/logstash.yml new file mode 100644 index 0000000000..3ddf63f9cc --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/logstash.yml @@ -0,0 +1,16 @@ +http.host: "0.0.0.0" +## Path where pipeline configurations reside +path.config: /usr/share/logstash/pipeline + +## Type of queue : memeory based or file based +#queue.type: persisted +## Size of queue +#queue.max_bytes: 1024mb +## Setting true makes logstash check periodically for change in pipeline configurations +config.reload.automatic: true + +## xpack configurations +#xpack.monitoring.elasticsearch.url: ["http://10.247.186.12:9200", "http://10.247.186.13:9200"] +#xpack.monitoring.elasticsearch.username: elastic +#xpack.monitoring.elasticsearch.password: changeme +xpack.monitoring.enabled: false diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf new file mode 100644 index 0000000000..aa087e3445 --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf @@ -0,0 +1,108 @@ +input { + http_poller { + urls => { + event_queue => { + method => get + url => "${dmaap_base_url}/events/${event_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + add_field => { "topic" => "${event_topic}" } + } + notification_queue => { + method => get + url => "${dmaap_base_url}/events/${notification_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + add_field => { "topic" => "${notification_topic}" } + } + request_queue => { + method => get + url => "${dmaap_base_url}/events/${request_topic}/${dmaap_consumer_group}/${dmaap_consumer_id}?timeout=15000" + headers => { + Accept => "application/json" + } + add_field => { "topic" => "${request_topic}" } + } + } + socket_timeout => 30 + request_timeout => 30 + interval => 60 + codec => "plain" + } +} + +filter { + # avoid noise if no entry in the list + if [message] == "[]" { + drop { } + } + + # parse json, split the list into multiple events, and parse each event + json { + source => "[message]" + target => "message" + } + split { + field => "message" + } + json { + source => "message" + } + mutate { remove_field => [ "message" ] } + # express timestamps in milliseconds instead of microseconds + ruby { + code => "event.set('closedLoopAlarmStart', Integer(event.get('closedLoopAlarmStart')))" + } + date { + match => [ "closedLoopAlarmStart", UNIX_MS ] + target => "closedLoopAlarmStart" + } + + if [closedLoopAlarmEnd] { + ruby { + code => "event.set('closedLoopAlarmEnd', Integer(event.get('closedLoopAlarmEnd')))" + } + date { + match => [ "closedLoopAlarmEnd", UNIX_MS ] + target => "closedLoopAlarmEnd" + } + + } + #"yyyy-MM-dd HH:mm:ss" + if [notificationTime] { + mutate { + gsub => [ + "notificationTime", " ", "T" + ] + } + date { + match => [ "notificationTime", ISO8601 ] + target => "notificationTime" + } + } +} +output { + stdout { + codec => rubydebug + } + + if [http_request_failure] { + elasticsearch { + codec => "json" + hosts => ["${elasticsearch_base_url}"] + index => "errors-%{+YYYY.MM.DD}" + doc_as_upsert => true + } + } else { + elasticsearch { + codec => "json" + hosts => ["${elasticsearch_base_url}"] + index => "logstash-%{+YYYY.MM.DD}" # creates daily indexes + doc_as_upsert => true + + } + } + +} diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/templates/configmap.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/templates/configmap.yaml new file mode 100644 index 0000000000..7339832f36 --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/templates/configmap.yaml @@ -0,0 +1,23 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/templates/deployment.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/templates/deployment.yaml new file mode 100644 index 0000000000..3cfdae3aeb --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/templates/deployment.yaml @@ -0,0 +1,119 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} + spec: + initContainers: + - command: + - /root/ready.py + args: + - --container-name + - clamp-dash-es + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-readiness + containers: + - name: {{ include "common.name" . }} + image: "{{ .Values.global.loggingRepository| default .Values.loggingRepository }}/{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + env: + - name: dmaap_consumer_group + value: "{{ .Values.config.dmaapConsumerGroup }}" + - name: dmaap_consumer_id + value: "{{ .Values.config.dmaapConsumerId }}" + - name: event_topic + value: "{{ .Values.config.eventTopic }}" + - name: notification_topic + value: "{{ .Values.config.notificationTopic }}" + - name: request_topic + value: "{{ .Values.config.requestTopic }}" + - name: dmaap_base_url + value: {{ .Values.config.dmaapScheme }}://{{ .Values.config.dmaapHost }}.{{ include "common.namespace" . }}:{{ .Values.config.dmaapPort }} + - name: elasticsearch_base_url + value: "http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}.svc.cluster.local:{{.Values.config.elasticsearchPort}}" + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ include "common.servicename" . }} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} +# disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end -}} + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /usr/share/logstash/config/logstash.yml + name: {{ include "common.fullname" . }} + subPath: logstash.yml + - mountPath: /usr/share/logstash/pipeline/logstash.conf + name: {{ include "common.fullname" . }} + subPath: pipeline.conf + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 10 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 10 }} + {{- end }} + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }} + configMap: + name: {{ include "common.fullname" . }} + items: + - key: logstash.yml + path: logstash.yml + - key: pipeline.conf + path: pipeline.conf + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/templates/service.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/templates/service.yaml new file mode 100644 index 0000000000..9d0a51149e --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/templates/service.yaml @@ -0,0 +1,42 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.servicename" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.externalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.config.portName }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.config.portName }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml new file mode 100644 index 0000000000..bb12210398 --- /dev/null +++ b/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml @@ -0,0 +1,100 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + repositorySecret: eyJuZXh1czMub25hcC5vcmc6MTAwMDEiOnsidXNlcm5hbWUiOiJkb2NrZXIiLCJwYXNzd29yZCI6ImRvY2tlciIsImVtYWlsIjoiQCIsImF1dGgiOiJaRzlqYTJWeU9tUnZZMnRsY2c9PSJ9fQ== + readinessRepository: oomk8s + readinessImage: readiness-check:1.1.0 + persistence: {} + +################################################################# +# Application configuration defaults. +################################################################# + +# application image +loggingRepository: docker.elastic.co +image: logstash/logstash:5.6.9 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +config: + elasticsearchServiceName: cdash-es + elasticsearchPort: 9200 + dmaapHost: message-router.{{ include "common.namespace" . }} + dmaapScheme: https + dmaapPort: 3905 + dmaapConsumerGroup: "clampdashboard" + dmaapConsumerId: "clampdashboard" + eventTopic: "DCAE-CL-EVENT" + notificationTopic: "POLICY-CL-MGT" + requestTopic: "APPC-CL" + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 120 + periodSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 10 + periodSeconds: 10 + +service: + #Example service definition with external, internal and node ports. + #Services may use any combination of ports depending on the 'type' of + #service being defined. + type: ClusterIP + name: cdash-ls + portName: cdash-ls-healthcheck + externalPort: 9600 + internalPort: 9600 +ingress: + enabled: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # Example: + # Configure resource requests and limits + # ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # Minimum memory for development is 2 CPU cores and 4GB memory + # Minimum memory for production is 4 CPU cores and 8GB memory +#resources: +# limits: +# cpu: 2 +# memory: 4Gi +# requests: +# cpu: 2 +# memory: 4Gi |