From 9a994243c2da301f63270776bf613c329d1ff588 Mon Sep 17 00:00:00 2001 From: Sylvain Desbureaux Date: Sat, 21 Nov 2020 22:25:10 +0100 Subject: [LOG] Uses new tpls for repos / images This commit makes Log chart to use the new generator for repositories and images. Issue-ID: OOM-2364 Change-Id: Ie93c5fccc2c9ed6026cd8dd93a3d7e141d1bed83 Signed-off-by: Jakub Latusek --- kubernetes/log/Makefile | 56 +++++ kubernetes/log/charts/log-elasticsearch/Chart.yaml | 19 -- .../log/charts/log-elasticsearch/requirements.yaml | 19 -- .../resources/config/elasticsearch.yml | 146 ----------- .../charts/log-elasticsearch/templates/NOTES.txt | 33 --- .../log-elasticsearch/templates/configmap.yaml | 29 --- .../log-elasticsearch/templates/deployment.yaml | 112 --------- .../log-elasticsearch/templates/ingress.yaml | 1 - .../log/charts/log-elasticsearch/templates/pv.yaml | 37 --- .../charts/log-elasticsearch/templates/pvc.yaml | 38 --- .../log-elasticsearch/templates/service.yaml | 70 ------ .../log/charts/log-elasticsearch/values.yaml | 123 --------- kubernetes/log/charts/log-kibana/Chart.yaml | 19 -- kubernetes/log/charts/log-kibana/requirements.yaml | 19 -- .../charts/log-kibana/resources/config/README.txt | 1 - .../resources/config/kibana-onboarding.json | 0 .../charts/log-kibana/resources/config/kibana.yml | 129 ---------- .../log/charts/log-kibana/templates/NOTES.txt | 32 --- .../log/charts/log-kibana/templates/configmap.yaml | 29 --- .../charts/log-kibana/templates/deployment.yaml | 114 --------- .../log/charts/log-kibana/templates/ingress.yaml | 1 - .../log/charts/log-kibana/templates/service.yaml | 43 ---- kubernetes/log/charts/log-kibana/values.yaml | 107 -------- kubernetes/log/charts/log-logstash/Chart.yaml | 19 -- .../log/charts/log-logstash/requirements.yaml | 19 -- .../log-logstash/resources/config/logstash.yml | 32 --- .../resources/config/onap-pipeline.conf | 276 --------------------- .../log/charts/log-logstash/templates/NOTES.txt | 32 --- .../charts/log-logstash/templates/configmap.yaml | 29 --- .../charts/log-logstash/templates/deployment.yaml | 102 -------- .../log/charts/log-logstash/templates/ingress.yaml | 1 - .../log/charts/log-logstash/templates/service.yaml | 70 ------ kubernetes/log/charts/log-logstash/values.yaml | 104 -------- kubernetes/log/components/Makefile | 56 +++++ .../log/components/log-elasticsearch/Chart.yaml | 19 ++ .../components/log-elasticsearch/requirements.yaml | 22 ++ .../resources/config/elasticsearch.yml | 146 +++++++++++ .../log-elasticsearch/templates/NOTES.txt | 33 +++ .../log-elasticsearch/templates/configmap.yaml | 29 +++ .../log-elasticsearch/templates/deployment.yaml | 112 +++++++++ .../log-elasticsearch/templates/ingress.yaml | 1 + .../components/log-elasticsearch/templates/pv.yaml | 37 +++ .../log-elasticsearch/templates/pvc.yaml | 38 +++ .../log-elasticsearch/templates/service.yaml | 70 ++++++ .../log/components/log-elasticsearch/values.yaml | 116 +++++++++ kubernetes/log/components/log-kibana/Chart.yaml | 19 ++ .../log/components/log-kibana/requirements.yaml | 22 ++ .../log-kibana/resources/config/README.txt | 1 + .../resources/config/kibana-onboarding.json | 0 .../log-kibana/resources/config/kibana.yml | 129 ++++++++++ .../log/components/log-kibana/templates/NOTES.txt | 32 +++ .../components/log-kibana/templates/configmap.yaml | 29 +++ .../log-kibana/templates/deployment.yaml | 114 +++++++++ .../components/log-kibana/templates/ingress.yaml | 1 + .../components/log-kibana/templates/service.yaml | 43 ++++ kubernetes/log/components/log-kibana/values.yaml | 100 ++++++++ kubernetes/log/components/log-logstash/Chart.yaml | 19 ++ .../log/components/log-logstash/requirements.yaml | 22 ++ .../log-logstash/resources/config/logstash.yml | 32 +++ .../resources/config/onap-pipeline.conf | 276 +++++++++++++++++++++ .../components/log-logstash/templates/NOTES.txt | 32 +++ .../log-logstash/templates/configmap.yaml | 29 +++ .../log-logstash/templates/deployment.yaml | 102 ++++++++ .../components/log-logstash/templates/ingress.yaml | 1 + .../components/log-logstash/templates/service.yaml | 70 ++++++ kubernetes/log/components/log-logstash/values.yaml | 102 ++++++++ kubernetes/log/requirements.yaml | 13 + kubernetes/log/values.yaml | 1 - 68 files changed, 1923 insertions(+), 1806 deletions(-) create mode 100644 kubernetes/log/Makefile delete mode 100644 kubernetes/log/charts/log-elasticsearch/Chart.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/requirements.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/NOTES.txt delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/configmap.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/ingress.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/pv.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/pvc.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/templates/service.yaml delete mode 100644 kubernetes/log/charts/log-elasticsearch/values.yaml delete mode 100644 kubernetes/log/charts/log-kibana/Chart.yaml delete mode 100644 kubernetes/log/charts/log-kibana/requirements.yaml delete mode 100644 kubernetes/log/charts/log-kibana/resources/config/README.txt delete mode 100644 kubernetes/log/charts/log-kibana/resources/config/kibana-onboarding.json delete mode 100644 kubernetes/log/charts/log-kibana/resources/config/kibana.yml delete mode 100644 kubernetes/log/charts/log-kibana/templates/NOTES.txt delete mode 100644 kubernetes/log/charts/log-kibana/templates/configmap.yaml delete mode 100644 kubernetes/log/charts/log-kibana/templates/deployment.yaml delete mode 100644 kubernetes/log/charts/log-kibana/templates/ingress.yaml delete mode 100644 kubernetes/log/charts/log-kibana/templates/service.yaml delete mode 100644 kubernetes/log/charts/log-kibana/values.yaml delete mode 100644 kubernetes/log/charts/log-logstash/Chart.yaml delete mode 100644 kubernetes/log/charts/log-logstash/requirements.yaml delete mode 100644 kubernetes/log/charts/log-logstash/resources/config/logstash.yml delete mode 100644 kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf delete mode 100644 kubernetes/log/charts/log-logstash/templates/NOTES.txt delete mode 100644 kubernetes/log/charts/log-logstash/templates/configmap.yaml delete mode 100644 kubernetes/log/charts/log-logstash/templates/deployment.yaml delete mode 100644 kubernetes/log/charts/log-logstash/templates/ingress.yaml delete mode 100644 kubernetes/log/charts/log-logstash/templates/service.yaml delete mode 100644 kubernetes/log/charts/log-logstash/values.yaml create mode 100644 kubernetes/log/components/Makefile create mode 100644 kubernetes/log/components/log-elasticsearch/Chart.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/requirements.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/resources/config/elasticsearch.yml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/NOTES.txt create mode 100644 kubernetes/log/components/log-elasticsearch/templates/configmap.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/deployment.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/ingress.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/pv.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/pvc.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/templates/service.yaml create mode 100644 kubernetes/log/components/log-elasticsearch/values.yaml create mode 100644 kubernetes/log/components/log-kibana/Chart.yaml create mode 100644 kubernetes/log/components/log-kibana/requirements.yaml create mode 100644 kubernetes/log/components/log-kibana/resources/config/README.txt create mode 100644 kubernetes/log/components/log-kibana/resources/config/kibana-onboarding.json create mode 100644 kubernetes/log/components/log-kibana/resources/config/kibana.yml create mode 100644 kubernetes/log/components/log-kibana/templates/NOTES.txt create mode 100644 kubernetes/log/components/log-kibana/templates/configmap.yaml create mode 100644 kubernetes/log/components/log-kibana/templates/deployment.yaml create mode 100644 kubernetes/log/components/log-kibana/templates/ingress.yaml create mode 100644 kubernetes/log/components/log-kibana/templates/service.yaml create mode 100644 kubernetes/log/components/log-kibana/values.yaml create mode 100644 kubernetes/log/components/log-logstash/Chart.yaml create mode 100644 kubernetes/log/components/log-logstash/requirements.yaml create mode 100644 kubernetes/log/components/log-logstash/resources/config/logstash.yml create mode 100644 kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf create mode 100644 kubernetes/log/components/log-logstash/templates/NOTES.txt create mode 100644 kubernetes/log/components/log-logstash/templates/configmap.yaml create mode 100644 kubernetes/log/components/log-logstash/templates/deployment.yaml create mode 100644 kubernetes/log/components/log-logstash/templates/ingress.yaml create mode 100644 kubernetes/log/components/log-logstash/templates/service.yaml create mode 100644 kubernetes/log/components/log-logstash/values.yaml (limited to 'kubernetes') diff --git a/kubernetes/log/Makefile b/kubernetes/log/Makefile new file mode 100644 index 0000000000..89b2f465ec --- /dev/null +++ b/kubernetes/log/Makefile @@ -0,0 +1,56 @@ +# Copyright © 2020 Samsung Electronics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +OUTPUT_DIR := $(ROOT_DIR)/../dist +PACKAGE_DIR := $(OUTPUT_DIR)/packages +SECRET_DIR := $(OUTPUT_DIR)/secrets + +EXCLUDES := dist resources templates charts docker +HELM_BIN := helm +HELM_CHARTS := $(filter-out $(EXCLUDES), $(sort $(patsubst %/.,%,$(wildcard */.)))) +HELM_VER := $(shell $(HELM_BIN) version --template "{{.Version}}") + +.PHONY: $(EXCLUDES) $(HELM_CHARTS) + +all: $(HELM_CHARTS) + +$(HELM_CHARTS): + @echo "\n[$@]" + @make package-$@ + +make-%: + @if [ -f $*/Makefile ]; then make -C $*; fi + +dep-%: make-% + @if [ -f $*/requirements.yaml ]; then $(HELM_BIN) dep up $*; fi + +lint-%: dep-% + @if [ -f $*/Chart.yaml ]; then $(HELM_BIN) lint $*; fi + +package-%: lint-% + @mkdir -p $(PACKAGE_DIR) +ifeq "$(findstring v3,$(HELM_VER))" "v3" + @if [ -f $*/Chart.yaml ]; then PACKAGE_NAME=$$($(HELM_BIN) package -d $(PACKAGE_DIR) $* | cut -d":" -f2) && $(HELM_BIN) push -f $$PACKAGE_NAME local; fi +else + @if [ -f $*/Chart.yaml ]; then $(HELM_BIN) package -d $(PACKAGE_DIR) $*; fi +endif + @$(HELM_BIN) repo index $(PACKAGE_DIR) + +clean: + @rm -f */requirements.lock + @rm -f *tgz */charts/*tgz + @rm -rf $(PACKAGE_DIR) +%: + @: diff --git a/kubernetes/log/charts/log-elasticsearch/Chart.yaml b/kubernetes/log/charts/log-elasticsearch/Chart.yaml deleted file mode 100644 index 66b0257e35..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/Chart.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: ONAP Logging Elasticsearch -name: log-elasticsearch -version: 6.0.0 diff --git a/kubernetes/log/charts/log-elasticsearch/requirements.yaml b/kubernetes/log/charts/log-elasticsearch/requirements.yaml deleted file mode 100644 index caff1e5dc4..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/requirements.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -dependencies: - - name: common - version: ~6.x-0 - repository: '@local' diff --git a/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml b/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml deleted file mode 100644 index d39fc97e0d..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html -# -# ---------------------------------- Cluster ----------------------------------- -# -# Name of the Elasticsearch cluster. -# A node can only join a cluster when it shares its cluster.name with all the other nodes in the cluster. -# The default name is elasticsearch, but you should change it to an appropriate name which describes the -# purpose of the cluster. -# -cluster.name: "onap-log" -# -# The port that other nodes in the cluster should use when communicating with this node. -# Required for Elasticsearch's nodes running on different cluster nodes. -# More : https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html -#transport.publish_port:$transport.publish_port -# -# The host address to publish for nodes in the cluster to connect to. -# Required for Elasticsearch's nodes running on different cluster nodes. -# More : https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html -#transport.publish_host:$transport.publish_host -# -# ------------------------------------ Node ------------------------------------ -# -# It is better to provide different meaningfull names fot different elastic nodes. -# By default, Elasticsearch will take the 7 first character of the randomly generated uuid used as the node id. -# Note that the node id is persisted and does not change when a node restarts -# -#node.name: $node.name -# -# Add custom attributes to the node: -# -#node.attr.rack: r1 -# -# ----------------------------------- Paths ------------------------------------ -# -# The location of the data files of each index / shard allocated on the node. Can hold multiple locations separated by coma. -# In production, we should not keep this default to "/elasticsearch/data", as on upgrading Elasticsearch, directory structure -# may change & can deal to data loss. -path.data: /usr/share/elasticsearch/data -# -# Elasticsearch's log files location. In production, we should not keep this default to "/elasticsearch/logs", -# as on upgrading Elasticsearch, directory structure may change. -path.logs: /usr/share/elasticsearch/logs -# -# ----------------------------------- Memory ----------------------------------- -# -# It is vitally important to the health of your node that none of the JVM is ever swapped out to disk. -# Lock the memory on startup. -# -bootstrap.memory_lock: false -# -# Make sure that the heap size is set to about half the memory available -# on the system and that the owner of the process is allowed to use this -# limit. -# -# Elasticsearch performs poorly when the system is swapping the memory. -# -# ---------------------------------- Network ----------------------------------- -# -# Set the bind address to a specific IP (IPv4 or IPv6): -# In order to communicate and to form a cluster with nodes on other servers, your node will need to bind to a -# non-loopback address. -network.host: 0.0.0.0 -# -# Set a custom port for HTTP: If required, default is 9200-9300 -# -#http.port: $http.port -# -# For more information, consult the network module documentation. -# -# --------------------------------- Discovery ---------------------------------- -# -# Pass an initial list of hosts to perform discovery when new node is started -# To form a cluster with nodes on other servers, you have to provide a seed list of other nodes in the cluster -# that are likely to be live and contactable. -# By default, Elasticsearch will bind to the available loopback addresses and will scan ports 9300 to 9305 to try -# to connect to other nodes running on the same server. -# -#$discovery.zen.ping.unicast.hosts -# -# This setting tells Elasticsearch to not elect a master unless there are enough master-eligible nodes -# available. Only then will an election take place. -# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): -discovery.zen.minimum_master_nodes: 1 -# -# For more information, consult the zen discovery module documentation. -# -# ---------------------------------- Gateway ----------------------------------- -# -# Block initial recovery after a full cluster restart until N nodes are started: -# -#gateway.recover_after_nodes: 3 -# -# For more information, consult the gateway module documentation. -# -# ---------------------------------- Various ----------------------------------- -# -# Require explicit names when deleting indices: -# -#action.destructive_requires_name: true -# Set a custom port for HTTP: If required, default is 9200-9300 -# This is used for REST APIs -http.port: {{.Values.service.externalPort}} -# Port to bind for communication between nodes. Accepts a single value or a range. -# If a range is specified, the node will bind to the first available port in the range. -# Defaults to 9300-9400. -# More info: -transport.tcp.port: {{.Values.service.externalPort2}} - -xpack.graph.enabled: false -#Set to false to disable X-Pack graph features. - -xpack.ml.enabled: false -#Set to false to disable X-Pack machine learning features. - -xpack.monitoring.enabled: false -#Set to false to disable X-Pack monitoring features. - -xpack.security.enabled: false -#Set to false to disable X-Pack security features. - -xpack.watcher.enabled: false -#Set to false to disable Watcher. diff --git a/kubernetes/log/charts/log-elasticsearch/templates/NOTES.txt b/kubernetes/log/charts/log-elasticsearch/templates/NOTES.txt deleted file mode 100644 index ab908cd309..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/NOTES.txt +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range .Values.ingress.hosts }} - http://{{ . }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.externalPort }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} -{{- end }} diff --git a/kubernetes/log/charts/log-elasticsearch/templates/configmap.yaml b/kubernetes/log/charts/log-elasticsearch/templates/configmap.yaml deleted file mode 100644 index fe0349ede9..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/configmap.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }}-configmap - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -data: -{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml b/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml deleted file mode 100644 index 05a9b18ac9..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml +++ /dev/null @@ -1,112 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - template: - metadata: - labels: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} - spec: - initContainers: - - command: - - /bin/sh - - -c - - | - sysctl -w vm.max_map_count=262144 - mkdir -p /logroot/elasticsearch/logs - mkdir -p /logroot/elasticsearch/data - chmod -R 777 /logroot/elasticsearch - chown -R root:root /logroot - env: - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - securityContext: - privileged: true - image: {{ .Values.global.busyboxRepository | default .Values.busyboxRepository }}/{{ .Values.busyboxImage }} - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - name: init-sysctl - volumeMounts: - - name: {{ include "common.fullname" . }}-logs - mountPath: /logroot/ - containers: - - name: {{ include "common.name" . }} - image: "{{ .Values.global.loggingRepository | default .Values.loggingRepository }}/{{ .Values.image }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - resources: -{{ include "common.resources" . | indent 12 }} - ports: - - containerPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - - containerPort: {{ .Values.service.internalPort2 }} - name: {{ .Values.service.name2 }} -# disable liveness probe when breakpoints set in debugger - # so K8s doesn't restart unresponsive container - {{- if eq .Values.liveness.enabled true }} - livenessProbe: - tcpSocket: - port: {{ .Values.service.internalPort }} - initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.liveness.periodSeconds }} - {{ end -}} - readinessProbe: - tcpSocket: - port: {{ .Values.service.internalPort2 }} - initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.readiness.periodSeconds }} - env: - volumeMounts: - - mountPath: /etc/localtime - name: localtime - readOnly: true - - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml - name: {{ include "common.fullname" . }}-config - subPath: elasticsearch.yml - - mountPath: /usr/share/elasticsearch/data/ - name: {{ include "common.fullname" . }}-data - volumes: - - name: localtime - hostPath: - path: /etc/localtime - - name: {{ include "common.fullname" . }}-config - configMap: - name: {{ include "common.fullname" . }}-configmap - items: - - key: elasticsearch.yml - path: elasticsearch.yml - - name: {{ include "common.fullname" . }}-data - persistentVolumeClaim: - claimName: {{ include "common.fullname" . }} - - name: {{ include "common.fullname" . }}-logs - hostPath: - path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ include "common.release" . }}/{{ .Values.persistence.mountSubPathLogs }} - imagePullSecrets: - - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/charts/log-elasticsearch/templates/ingress.yaml b/kubernetes/log/charts/log-elasticsearch/templates/ingress.yaml deleted file mode 100644 index 8f87c68f1e..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/ingress.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.ingress" . }} diff --git a/kubernetes/log/charts/log-elasticsearch/templates/pv.yaml b/kubernetes/log/charts/log-elasticsearch/templates/pv.yaml deleted file mode 100644 index 9d4093db11..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/pv.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{/* -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} -kind: PersistentVolume -apiVersion: v1 -metadata: - name: {{ include "common.fullname" . }}-data - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - release: "{{ include "common.release" . }}" - heritage: "{{ .Release.Service }}" - name: {{ include "common.fullname" . }} -spec: - capacity: - storage: {{ .Values.persistence.size}} - accessModes: - - {{ .Values.persistence.accessMode }} - persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }} - storageClassName: "{{ include "common.fullname" . }}-data" - hostPath: - path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ include "common.release" . }}/{{ .Values.persistence.mountSubPath }} -{{- end -}} diff --git a/kubernetes/log/charts/log-elasticsearch/templates/pvc.yaml b/kubernetes/log/charts/log-elasticsearch/templates/pvc.yaml deleted file mode 100644 index 6ae4eea0d3..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/pvc.yaml +++ /dev/null @@ -1,38 +0,0 @@ -{{/* -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ include "common.release" . }}" - heritage: "{{ .Release.Service }}" -{{- if .Values.persistence.annotations }} - annotations: -{{ toYaml .Values.persistence.annotations | indent 4 }} -{{- end }} -spec: - accessModes: - - {{ .Values.persistence.accessMode }} - storageClassName: {{ include "common.storageClass" . }} - resources: - requests: - storage: {{ .Values.persistence.size }} -{{- end -}} diff --git a/kubernetes/log/charts/log-elasticsearch/templates/service.yaml b/kubernetes/log/charts/log-elasticsearch/templates/service.yaml deleted file mode 100644 index 7736f0c9d7..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/templates/service.yaml +++ /dev/null @@ -1,70 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} - annotations: -spec: - type: {{ .Values.service.type }} - ports: - {{if eq .Values.service.type "NodePort" -}} - - port: {{ .Values.service.internalPort }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} - name: {{ .Values.service.name }} - {{- else -}} - - port: {{ .Values.service.externalPort }} - targetPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - {{- end}} - selector: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name2 }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} - annotations: -spec: - type: {{ .Values.service.type2 }} - ports: - {{if eq .Values.service.type2 "NodePort" -}} - - port: {{ .Values.service.internalPort2 }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} - name: {{ .Values.service.name2 }} - {{- else -}} - - port: {{ .Values.service.externalPort2 }} - targetPort: {{ .Values.service.internalPort2 }} - name: {{ .Values.service.name2 }} - {{- end}} - selector: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} diff --git a/kubernetes/log/charts/log-elasticsearch/values.yaml b/kubernetes/log/charts/log-elasticsearch/values.yaml deleted file mode 100644 index 2e4f962d23..0000000000 --- a/kubernetes/log/charts/log-elasticsearch/values.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################# -# Global configuration defaults. -################################################################# -global: - nodePortPrefix: 302 - loggingRepository: docker.elastic.co - loggingImage: beats/filebeat:5.5.0 - persistence: {} - -################################################################# -# Application configuration defaults. -################################################################# - -# BusyBox image -busyboxRepository: docker.io -busyboxImage: library/busybox:latest - -# application image -loggingRepository: docker.elastic.co -image: elasticsearch/elasticsearch:5.5.0 -pullPolicy: Always - -# flag to enable debugging - application support required -debugEnabled: false - -# application configuration -# Example: -config: {} - -# default number of instances -replicaCount: 1 - -nodeSelector: {} - -affinity: {} - -# probe configuration parameters -liveness: - initialDelaySeconds: 120 - periodSeconds: 10 - # necessary to disable liveness probe when setting breakpoints - # in debugger so K8s doesn't restart unresponsive container - enabled: true - -readiness: - initialDelaySeconds: 120 - periodSeconds: 10 - -## Persist data to a persitent volume -persistence: - enabled: true - - ## A manually managed Persistent Volume and Claim - ## Requires persistence.enabled: true - ## If defined, PVC must be created manually before volume will be bound - # existingClaim: - volumeReclaimPolicy: Retain - - ## database data Persistent Volume Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - accessMode: ReadWriteOnce - size: 1Gi - mountPath: /dockerdata-nfs - mountSubPath: log/elasticsearch/data - mountSubPathLogs: log - -service: - type: NodePort - name: log-es - externalPort: 9200 - internalPort: 9200 - nodePort: 54 - type2: ClusterIP - name2: log-es-tcp - externalPort2: 9300 - internalPort2: 9300 - -ingress: - enabled: false - service: - - baseaddr: "loges" - name: "log-es" - port: 9200 - config: - ssl: "none" -# Resource Limit flavor -By Default using small -flavor: small -# Segregation for Different environment (Small and Large) -resources: - small: - limits: - cpu: 1 - memory: 4Gi - requests: - cpu: 1 - memory: 2Gi - large: - limits: - cpu: 2 - memory: 8Gi - requests: - cpu: 1 - memory: 4Gi - unlimited: {} diff --git a/kubernetes/log/charts/log-kibana/Chart.yaml b/kubernetes/log/charts/log-kibana/Chart.yaml deleted file mode 100644 index 8f5d973177..0000000000 --- a/kubernetes/log/charts/log-kibana/Chart.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: ONAP Logging Kibana -name: log-kibana -version: 6.0.0 diff --git a/kubernetes/log/charts/log-kibana/requirements.yaml b/kubernetes/log/charts/log-kibana/requirements.yaml deleted file mode 100644 index caff1e5dc4..0000000000 --- a/kubernetes/log/charts/log-kibana/requirements.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -dependencies: - - name: common - version: ~6.x-0 - repository: '@local' diff --git a/kubernetes/log/charts/log-kibana/resources/config/README.txt b/kubernetes/log/charts/log-kibana/resources/config/README.txt deleted file mode 100644 index 2863c1d5e5..0000000000 --- a/kubernetes/log/charts/log-kibana/resources/config/README.txt +++ /dev/null @@ -1 +0,0 @@ -"kibana-onboarding.json" file contains initial setup of Kibana obtained using Elasticdump tool. \ No newline at end of file diff --git a/kubernetes/log/charts/log-kibana/resources/config/kibana-onboarding.json b/kubernetes/log/charts/log-kibana/resources/config/kibana-onboarding.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/kubernetes/log/charts/log-kibana/resources/config/kibana.yml b/kubernetes/log/charts/log-kibana/resources/config/kibana.yml deleted file mode 100644 index 377f3c7b65..0000000000 --- a/kubernetes/log/charts/log-kibana/resources/config/kibana.yml +++ /dev/null @@ -1,129 +0,0 @@ -{{/* -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -xpack.graph.enabled: false -#Set to false to disable X-Pack graph features. -xpack.ml.enabled: false -#Set to false to disable X-Pack machine learning features. -xpack.monitoring.enabled: false -#Set to false to disable X-Pack monitoring features. -xpack.reporting.enabled: false -#Set to false to disable X-Pack reporting features. -xpack.security.enabled: false -#Set to false to disable X-Pack security features. -xpack.watcher.enabled: false -#Set to false to disable Watcher. -# Kibana is served by a back end server. This setting specifies the port to use. -server.port: {{.Values.service.externalPort}} - -# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. -# The default is 'localhost', which usually means remote machines will not be able to connect. -# To allow connections from remote users, set this parameter to a non-loopback address. -server.host: "0" - -# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects -# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests -# to Kibana. This setting cannot end in a slash. -#server.basePath: "" - -# The maximum payload size in bytes for incoming server requests. -#server.maxPayloadBytes: 1048576 - -# The Kibana server's name. This is used for display purposes. -server.name: "Kibana" - -# The URL of the Elasticsearch instance to use for all your queries. -elasticsearch.url: "http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}" -# When this setting's value is true Kibana uses the hostname specified in the server.host -# setting. When the value of this setting is false, Kibana uses the hostname of the host -# that connects to this Kibana instance. -#elasticsearch.preserveHost: true - -# Kibana uses an index in Elasticsearch to store saved searches, visualizations and -# dashboards. Kibana creates a new index if the index doesn't already exist. -#kibana.index: ".kibana" - -# The default application to load. -#kibana.defaultAppId: "discover" - -# If your Elasticsearch is protected with basic authentication, these settings provide -# the username and password that the Kibana server uses to perform maintenance on the Kibana -# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which -# is proxied through the Kibana server. -elasticsearch.username: "elastic" -elasticsearch.password: "changeme" -# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. -# These settings enable SSL for outgoing requests from the Kibana server to the browser. -#server.ssl.enabled: $server_ssl_enabled -#server.ssl.certificate: $server_ssl_certificate -#server.ssl.key: $server_ssl_key - -# Optional settings that provide the paths to the PEM-format SSL certificate and key files. -# These files validate that your Elasticsearch backend uses the same key files. -#elasticsearch.ssl.certificate: $elasticsearch_ssl_certificate -#elasticsearch.ssl.key: $elasticsearch_ssl_key - -# Optional setting that enables you to specify a path to the PEM file for the certificate -# authority for your Elasticsearch instance. -#elasticsearch.ssl.certificateAuthorities: $elasticsearch_ssl_certificateAuthorities - -# To disregard the validity of SSL certificates, change this setting's value to 'none'. -#elasticsearch.ssl.verificationMode: $elasticsearch_ssl_verificationMode - -# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of -# the elasticsearch.requestTimeout setting. -#elasticsearch.pingTimeout: 1500 - -# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value -# must be a positive integer. -#elasticsearch.requestTimeout: 30000 - -# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side -# headers, set this value to [] (an empty list). -#elasticsearch.requestHeadersWhitelist: [ authorization ] - -# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten -# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. -#elasticsearch.customHeaders: {} - -# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. -#elasticsearch.shardTimeout: 0 - -# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. -#elasticsearch.startupTimeout: 5000 - -# Specifies the path where Kibana creates the process ID file. -#pid.file: /var/run/kibana.pid - -# Enables you specify a file where Kibana stores log output. -#logging.dest: stdout - -# Set the value of this setting to true to suppress all logging output. -#logging.silent: false - -# Set the value of this setting to true to suppress all logging output other than error messages. -#logging.quiet: false - -# Set the value of this setting to true to log all events, including system usage information -# and all requests. -#logging.verbose: false - -# Set the interval in milliseconds to sample system and process performance -# metrics. Minimum is 100ms. Defaults to 5000. -#ops.interval: 5000 - -# The default locale. This locale can be used in certain circumstances to substitute any missing -# translations. -#i18n.defaultLocale: "en" diff --git a/kubernetes/log/charts/log-kibana/templates/NOTES.txt b/kubernetes/log/charts/log-kibana/templates/NOTES.txt deleted file mode 100644 index f115eb6f23..0000000000 --- a/kubernetes/log/charts/log-kibana/templates/NOTES.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range .Values.ingress.hosts }} - http://{{ . }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.externalPort }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} -{{- end }} diff --git a/kubernetes/log/charts/log-kibana/templates/configmap.yaml b/kubernetes/log/charts/log-kibana/templates/configmap.yaml deleted file mode 100644 index 3e98246df1..0000000000 --- a/kubernetes/log/charts/log-kibana/templates/configmap.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -data: -{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/charts/log-kibana/templates/deployment.yaml b/kubernetes/log/charts/log-kibana/templates/deployment.yaml deleted file mode 100644 index 4b0dcf1d3d..0000000000 --- a/kubernetes/log/charts/log-kibana/templates/deployment.yaml +++ /dev/null @@ -1,114 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - template: - metadata: - labels: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} - spec: - initContainers: - - command: - - /app/ready.py - args: - - --container-name - - log-elasticsearch - env: - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: "{{ include "common.repository" . }}/{{ .Values.global.readinessImage }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - name: {{ include "common.name" . }}-readiness - - args: - - --input=/config/kibana-onboarding.json - - --output=http://{{.Values.config.elasticsearchServiceName}}.{{ include "common.namespace" . }}:{{.Values.config.elasticsearchPort}}/.kibana - env: - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: "{{ .Values.elasticdumpRepository }}/{{ .Values.elasticdumpImage }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - name: {{ include "common.name" . }}-elasticdump - volumeMounts: - - mountPath: /config/kibana-onboarding.json - name: {{ include "common.fullname" . }} - subPath: kibana-onboarding.json - containers: - - name: {{ include "common.name" . }} - image: "{{ .Values.global.loggingRepository | default .Values.loggingRepository }}/{{ .Values.image }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - resources: -{{ include "common.resources" . | indent 12 }} - ports: - - containerPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - readinessProbe: - httpGet: - path: "/" - port: {{ .Values.service.internalPort }} - initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} -# disable liveness probe when breakpoints set in debugger - # so K8s doesn't restart unresponsive container - {{- if eq .Values.liveness.enabled true }} - livenessProbe: - httpGet: - path: "/" - port: {{ .Values.service.internalPort }} - initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} - {{ end -}} - env: - volumeMounts: - - mountPath: /etc/localtime - name: localtime - readOnly: true - - mountPath: /usr/share/kibana/config/ - name: {{ include "common.fullname" . }} - volumes: - - name: localtime - hostPath: - path: /etc/localtime - - name: {{ include "common.fullname" . }} - configMap: - name: {{ include "common.fullname" . }} - items: - - key: kibana.yml - path: kibana.yml - - key: kibana-onboarding.json - path: kibana-onboarding.json - imagePullSecrets: - - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/charts/log-kibana/templates/ingress.yaml b/kubernetes/log/charts/log-kibana/templates/ingress.yaml deleted file mode 100644 index 8f87c68f1e..0000000000 --- a/kubernetes/log/charts/log-kibana/templates/ingress.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.ingress" . }} diff --git a/kubernetes/log/charts/log-kibana/templates/service.yaml b/kubernetes/log/charts/log-kibana/templates/service.yaml deleted file mode 100644 index c53dc03368..0000000000 --- a/kubernetes/log/charts/log-kibana/templates/service.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} - annotations: -spec: - type: {{ .Values.service.type }} - ports: - {{if eq .Values.service.type "NodePort" -}} - - port: {{ .Values.service.externalPort }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} - name: {{ .Values.service.name }} - {{- else -}} - - port: {{ .Values.service.externalPort }} - targetPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - {{- end}} - selector: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} diff --git a/kubernetes/log/charts/log-kibana/values.yaml b/kubernetes/log/charts/log-kibana/values.yaml deleted file mode 100644 index 5cb2fb0a1c..0000000000 --- a/kubernetes/log/charts/log-kibana/values.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################# -# Global configuration defaults. -################################################################# -global: - nodePortPrefix: 302 - readinessImage: onap/oom/readiness:3.0.1 - persistence: {} - -################################################################# -# Application configuration defaults. -################################################################# - -# Elasticdump image -elasticdumpRepository: docker.io -elasticdumpImage: taskrabbit/elasticsearch-dump - -# BusyBox image -busyboxRepository: docker.io -busyboxImage: library/busybox:latest - -# application image -loggingRepository: docker.elastic.co -image: kibana/kibana:5.5.0 -pullPolicy: Always - -# flag to enable debugging - application support required -debugEnabled: false - -# application configuration -config: - elasticsearchServiceName: log-es - elasticsearchPort: 9200 - -# default number of instances -replicaCount: 1 - -nodeSelector: {} - -affinity: {} - -# probe configuration parameters -liveness: - initialDelaySeconds: 300 - periodSeconds: 10 - timeoutSeconds: 1 - # necessary to disable liveness probe when setting breakpoints - # in debugger so K8s doesn't restart unresponsive container - enabled: true - -readiness: - initialDelaySeconds: 300 - periodSeconds: 10 - timeoutSeconds: 1 - -service: - #Example service definition with external, internal and node ports. - #Services may use any combination of ports depending on the 'type' of - #service being defined. - type: NodePort - name: log-kibana - externalPort: 5601 - internalPort: 5601 - nodePort: 53 - -ingress: - enabled: false - service: - - baseaddr: "kibana.api" - name: "log-kibana" - port: 5601 - config: - ssl: "none" - -# Resource Limit flavor -By Default using small -flavor: small -# Segregation for Different environment (Small and Large) -resources: - small: - limits: - cpu: 2 - memory: 4Gi - requests: - cpu: 1 - memory: 2Gi - large: - limits: - cpu: 4 - memory: 8Gi - requests: - cpu: 2 - memory: 4Gi - unlimited: {} diff --git a/kubernetes/log/charts/log-logstash/Chart.yaml b/kubernetes/log/charts/log-logstash/Chart.yaml deleted file mode 100644 index 8349548f86..0000000000 --- a/kubernetes/log/charts/log-logstash/Chart.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -description: ONAP Logging Logstash -name: log-logstash -version: 6.0.0 diff --git a/kubernetes/log/charts/log-logstash/requirements.yaml b/kubernetes/log/charts/log-logstash/requirements.yaml deleted file mode 100644 index caff1e5dc4..0000000000 --- a/kubernetes/log/charts/log-logstash/requirements.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -dependencies: - - name: common - version: ~6.x-0 - repository: '@local' diff --git a/kubernetes/log/charts/log-logstash/resources/config/logstash.yml b/kubernetes/log/charts/log-logstash/resources/config/logstash.yml deleted file mode 100644 index 7c3bd8f851..0000000000 --- a/kubernetes/log/charts/log-logstash/resources/config/logstash.yml +++ /dev/null @@ -1,32 +0,0 @@ -{{/* -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -http.host: "0.0.0.0" -pipeline.workers: 3 -## Path where pipeline configurations reside -path.config: /usr/share/logstash/pipeline - -## Type of queue : memeory based or file based -#queue.type: persisted -## Size of queue -#queue.max_bytes: 1024mb -## Setting true makes logstash check periodically for change in pipeline configurations -config.reload.automatic: true - -## xpack configurations -#xpack.monitoring.elasticsearch.url: ["http://10.247.186.12:9200", "http://10.247.186.13:9200"] -#xpack.monitoring.elasticsearch.username: elastic -#xpack.monitoring.elasticsearch.password: changeme -xpack.monitoring.enabled: false diff --git a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf b/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf deleted file mode 100644 index d6b0696b81..0000000000 --- a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf +++ /dev/null @@ -1,276 +0,0 @@ -{{/* -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} -input { - beats { - - ## Add a id to plugin configuration. Can be anything unique. - id => 'beats_plugin' - - ######## Connection configurations ######## - - ## The port to listen on. - port => {{.Values.service.externalPort}} - - ## Close Idle clients after the specified time in seconds. Default is 60 seconds - #client_inactivity_timeout => 60 - - ######## Security configurations ######## - - ## Enable encryption. Default false. - #ssl => $filebeat_ssl - - ## ssl certificate path. - #ssl_certificate => $filebeat_ssl_certificate - - ## SSL key to use. - #ssl_key => $filebeat_ssl_key - - ##SSL key passphrase to use. - #ssl_key_passphrase => $filebeat_ssl_key_passphrase - - ## Value can be any of: none, peer, force_peer. - #ssl_verify_mode => $filebeat_ssl_verify_mode - - ## Time in milliseconds for an incomplete ssl handshake to timeout. Default is 10000 ms. - #ssl_handshake_timeout => 10000 - include_codec_tag => false - } -} - - -filter { - grok { - break_on_match => false - match => { - "source" => ["/var/log/onap/(?[^/]+)/", - "/var/log/onap/%{GREEDYDATA:componentLogFile}" - ] - } - } - - # Filter for log4j xml events - if "" in [message] { - - #mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug - - #Filter to parse xml event and retrieve data - xml { - source => "message" - store_xml => false - remove_namespaces => true - target => "xml_content" - xpath => [ "/event/message/text()", "logmsg" , - "/event/@logger", "Logger", - "/event/@timestamp", "Timestamp", - "/event/@level", "loglevel", - "/event/@thread", "Thread", - "/event/throwable/text()", "Exceptionthrowable", - "/event/NDC/text()", "NDCs", - "/event/properties/data/@name","mdcname", - "/event/properties/data/@value","mdcvalue"] - - } - - #Ruby filter to iterate and separate MDCs into documents - ruby { - code => ' - $i = 0 - $num = 0 - if event.get("[mdcname]") - $num = event.get("[mdcname]").length - end - if $num != 0 - until $i > $num do - if event.get("[mdcname]").at($i) and event.get("[mdcvalue]").at($i) - event.set(event.get("[mdcname]").at($i), event.get("[mdcvalue]").at($i)) - end - $i=$i+1 - end - end - ' - } - - #Validations - if [Exceptionthrowable] - { - mutate { - replace => { - "exceptionmessage" => "%{[Exceptionthrowable]}" - } - } - } - - if [NDCs] - { - mutate { - replace => { - "NDC" => "%{[NDCs]}" - } - } - } - - mutate { - replace => { - "Logger" =>"%{[Logger]}" - "logmsg" =>"%{[logmsg]}" - "Timestamp" =>"%{[Timestamp]}" - "loglevel" =>"%{[loglevel]}" - "message" => "%{logmsg}" - "Thread" => "%{[Thread]}" - } - remove_field => ["mdcname", "mdcvalue", "logmsg","Exceptionthrowable","NDCs"] - } - - if [Timestamp] - { - date { - match => ["Timestamp", "UNIX_MS"] - target => "Timestamp" - } - } - } - # Filter for logback events - else { - - #mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug - - mutate { - gsub => [ - 'message', ' = ', '=', - 'message', '= ', '=null', - 'message', '=\t', '=null ', #This null is followed by a tab - 'message', '\t$', '\t' - ] - } - # The grok below parses the message field for all current logback patterns used by oom components. - # Example logback pattern: %d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg - # Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message} - # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq - grok { - match => { - "message" => [ - "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}", - "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}", - "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}" - ] - } - overwrite => ["message"] - } - # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values. - kv { - source => "MDCs" - field_split => ",\t" - trim_key => "\s" - trim_value => "\s" - remove_field => [ "MDCs" ] - } - - if (![Timestamp] and [EndTimestamp]) { - mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } } - } - date { - match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ] - target => "Timestamp" - } - - mutate { - remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"] - } - - if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") { - #Parse kvps in message - kv { - field_split => "\s" - trim_key => "\s" - trim_value => "\s" - } - - #If Request Id is missing and DID is present use as RequestId - if (![RequestId] and [DID] =~ /.+/) { - mutate { add_field => { "RequestId" => "%{DID}" } } - } - } - - } #Close else statement for logback events -} #Close filter - - -output { - elasticsearch { - id => 'onap_es' - - ######### Security configurations ######### - - user => "elastic" - password => "changeme" - - ## The .cer or .pem file to validate the server's certificate - #cacert => $es_cacert - - ## The keystore used to present a certificate to the server. It can be either .jks or .p12 - #keystore => $es_keystore - #keystore_password => $es_keystore_password - - ## Enable SSL/TLS secured communication to Elasticsearch cluster. - ## Default is not set which in that case depends on the protocol specidfied in hosts list - #ssl => $es_ssl - - ## Option to validate the server's certificate. Default is true - #ssl_certificate_verification => $es_ssl_certificate_verification - - ## The JKS truststore to validate the server's certificate. - #truststore => $es_truststore - #truststore_password => $es_truststore_password - - - ######### Elasticsearchcluster and host configurations ######### - - ##can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered - hosts => ["http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}"] - - - ## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false. - sniffing => true - - ## How long to wait, in seconds, between sniffing attempts. Default is 5 seconds. - #sniffing_delay => 5 - - ## Set the address of a forward HTTP proxy. - #proxy => $es_proxy - - ##Use this if you must run Elasticsearch behind a proxy that remaps the root path for the Elasticsearch HTTP API lives - #path => $es_path - - ######### Elasticsearch request configurations ######### - - ## This setting defines the maximum sized bulk request Logstash will make. - #flush_size => ? - - ######### Document configurations ######### - - index => "logstash-%{+YYYY.MM.dd}" - document_type => "logs" - - ## This can be used to associate child documents with a parent using the parent ID. - #parent => "abcd' - } -} - diff --git a/kubernetes/log/charts/log-logstash/templates/NOTES.txt b/kubernetes/log/charts/log-logstash/templates/NOTES.txt deleted file mode 100644 index f115eb6f23..0000000000 --- a/kubernetes/log/charts/log-logstash/templates/NOTES.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range .Values.ingress.hosts }} - http://{{ . }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.externalPort }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} -{{- end }} diff --git a/kubernetes/log/charts/log-logstash/templates/configmap.yaml b/kubernetes/log/charts/log-logstash/templates/configmap.yaml deleted file mode 100644 index 3e98246df1..0000000000 --- a/kubernetes/log/charts/log-logstash/templates/configmap.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -data: -{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/charts/log-logstash/templates/deployment.yaml b/kubernetes/log/charts/log-logstash/templates/deployment.yaml deleted file mode 100644 index dde6c5bc8f..0000000000 --- a/kubernetes/log/charts/log-logstash/templates/deployment.yaml +++ /dev/null @@ -1,102 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: {{ include "common.fullname" . }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - template: - metadata: - labels: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} - spec: - initContainers: - - command: - - /app/ready.py - args: - - --container-name - - log-elasticsearch - env: - - name: NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: "{{ include "common.repository" . }}/{{ .Values.global.readinessImage }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - name: {{ include "common.name" . }}-readiness - containers: - - name: {{ include "common.name" . }} - image: "{{ .Values.global.loggingRepository| default .Values.loggingRepository }}/{{ .Values.image }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - resources: -{{ include "common.resources" . | indent 12 }} - ports: - - containerPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - - containerPort: {{ .Values.service.internalPort2 }} - name: {{ .Values.service.name2 }} - readinessProbe: - tcpSocket: - port: {{ .Values.service.internalPort }} - initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.readiness.periodSeconds }} -# disable liveness probe when breakpoints set in debugger - # so K8s doesn't restart unresponsive container - {{- if eq .Values.liveness.enabled true }} - livenessProbe: - tcpSocket: - port: {{ .Values.service.internalPort }} - initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.liveness.periodSeconds }} - {{ end -}} - env: - volumeMounts: - - mountPath: /etc/localtime - name: localtime - readOnly: true - - mountPath: /usr/share/logstash/config/ - name: {{ include "common.fullname" . }}-config - - mountPath: /usr/share/logstash/pipeline/ - name: {{ include "common.fullname" . }}-pipeline - volumes: - - name: localtime - hostPath: - path: /etc/localtime - - name: {{ include "common.fullname" . }}-config - configMap: - name: {{ include "common.fullname" . }} - items: - - key: logstash.yml - path: logstash.yml - - name: {{ include "common.fullname" . }}-pipeline - configMap: - name: {{ include "common.fullname" . }} - items: - - key: onap-pipeline.conf - path: onap-pipeline.conf - imagePullSecrets: - - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/charts/log-logstash/templates/ingress.yaml b/kubernetes/log/charts/log-logstash/templates/ingress.yaml deleted file mode 100644 index 8f87c68f1e..0000000000 --- a/kubernetes/log/charts/log-logstash/templates/ingress.yaml +++ /dev/null @@ -1 +0,0 @@ -{{ include "common.ingress" . }} diff --git a/kubernetes/log/charts/log-logstash/templates/service.yaml b/kubernetes/log/charts/log-logstash/templates/service.yaml deleted file mode 100644 index 7736f0c9d7..0000000000 --- a/kubernetes/log/charts/log-logstash/templates/service.yaml +++ /dev/null @@ -1,70 +0,0 @@ -{{/* -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} - annotations: -spec: - type: {{ .Values.service.type }} - ports: - {{if eq .Values.service.type "NodePort" -}} - - port: {{ .Values.service.internalPort }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} - name: {{ .Values.service.name }} - {{- else -}} - - port: {{ .Values.service.externalPort }} - targetPort: {{ .Values.service.internalPort }} - name: {{ .Values.service.name }} - {{- end}} - selector: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name2 }} - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} - annotations: -spec: - type: {{ .Values.service.type2 }} - ports: - {{if eq .Values.service.type2 "NodePort" -}} - - port: {{ .Values.service.internalPort2 }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} - name: {{ .Values.service.name2 }} - {{- else -}} - - port: {{ .Values.service.externalPort2 }} - targetPort: {{ .Values.service.internalPort2 }} - name: {{ .Values.service.name2 }} - {{- end}} - selector: - app: {{ include "common.name" . }} - release: {{ include "common.release" . }} diff --git a/kubernetes/log/charts/log-logstash/values.yaml b/kubernetes/log/charts/log-logstash/values.yaml deleted file mode 100644 index 483d1b721a..0000000000 --- a/kubernetes/log/charts/log-logstash/values.yaml +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright © 2017 Amdocs, Bell Canada -# Modifications Copyright © 2018 AT&T -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################# -# Global configuration defaults. -################################################################# -global: - nodePortPrefix: 302 - readinessImage: onap/oom/readiness:3.0.1 - persistence: {} - -################################################################# -# Application configuration defaults. -################################################################# - -# application image -loggingRepository: docker.elastic.co -image: logstash/logstash:5.4.3 -pullPolicy: Always - -# flag to enable debugging - application support required -debugEnabled: false - -# application configuration -config: - elasticsearchServiceName: log-es - elasticsearchPort: 9200 - -# default number of instances -# 30+ logs/sec will saturate a single node to 6+ vCores -replicaCount: 3 - -nodeSelector: {} - -affinity: {} - -# probe configuration parameters -liveness: - initialDelaySeconds: 120 - periodSeconds: 10 - # necessary to disable liveness probe when setting breakpoints - # in debugger so K8s doesn't restart unresponsive container - enabled: true - -readiness: - initialDelaySeconds: 10 - periodSeconds: 10 - -service: - #Example service definition with external, internal and node ports. - #Services may use any combination of ports depending on the 'type' of - #service being defined. - type: NodePort - name: log-ls - externalPort: 5044 - internalPort: 5044 - nodePort: 55 - type2: ClusterIP - name2: log-ls-http - externalPort2: 9600 - internalPort2: 9600 -ingress: - enabled: false - service: - - baseaddr: "logls" - name: "log-ls" - port: 5044 - - baseaddr: "loglshttp" - name: "log-ls" - port: 9600 - config: - ssl: "none" - -# Resource Limit flavor -By Default using small -flavor: small -# Segregation for Different environment (Small and Large) -resources: - small: - limits: - cpu: 1 - memory: 2Gi - requests: - cpu: 0.5 - memory: 1Gi - large: - limits: - cpu: 2 - memory: 4Gi - requests: - cpu: 2 - memory: 4Gi - unlimited: {} diff --git a/kubernetes/log/components/Makefile b/kubernetes/log/components/Makefile new file mode 100644 index 0000000000..d62cb0b700 --- /dev/null +++ b/kubernetes/log/components/Makefile @@ -0,0 +1,56 @@ +# Copyright © 2020 Samsung Electronics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +OUTPUT_DIR := $(ROOT_DIR)/../../dist +PACKAGE_DIR := $(OUTPUT_DIR)/packages +SECRET_DIR := $(OUTPUT_DIR)/secrets + +EXCLUDES := +HELM_BIN := helm +HELM_CHARTS := $(filter-out $(EXCLUDES), $(sort $(patsubst %/.,%,$(wildcard */.)))) +HELM_VER := $(shell $(HELM_BIN) version --template "{{.Version}}") + +.PHONY: $(EXCLUDES) $(HELM_CHARTS) + +all: $(HELM_CHARTS) + +$(HELM_CHARTS): + @echo "\n[$@]" + @make package-$@ + +make-%: + @if [ -f $*/Makefile ]; then make -C $*; fi + +dep-%: make-% + @if [ -f $*/requirements.yaml ]; then $(HELM_BIN) dep up $*; fi + +lint-%: dep-% + @if [ -f $*/Chart.yaml ]; then $(HELM_BIN) lint $*; fi + +package-%: lint-% + @mkdir -p $(PACKAGE_DIR) +ifeq "$(findstring v3,$(HELM_VER))" "v3" + @if [ -f $*/Chart.yaml ]; then PACKAGE_NAME=$$($(HELM_BIN) package -d $(PACKAGE_DIR) $* | cut -d":" -f2) && $(HELM_BIN) push -f $$PACKAGE_NAME local; fi +else + @if [ -f $*/Chart.yaml ]; then $(HELM_BIN) package -d $(PACKAGE_DIR) $*; fi +endif + @$(HELM_BIN) repo index $(PACKAGE_DIR) + +clean: + @rm -f */requirements.lock + @rm -f *tgz */charts/*tgz + @rm -rf $(PACKAGE_DIR) +%: + @: diff --git a/kubernetes/log/components/log-elasticsearch/Chart.yaml b/kubernetes/log/components/log-elasticsearch/Chart.yaml new file mode 100644 index 0000000000..66b0257e35 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/Chart.yaml @@ -0,0 +1,19 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: ONAP Logging Elasticsearch +name: log-elasticsearch +version: 6.0.0 diff --git a/kubernetes/log/components/log-elasticsearch/requirements.yaml b/kubernetes/log/components/log-elasticsearch/requirements.yaml new file mode 100644 index 0000000000..bdc19209e7 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/requirements.yaml @@ -0,0 +1,22 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~6.x-0 + repository: '@local' + - name: repositoryGenerator + version: ~6.x-0 + repository: '@local' diff --git a/kubernetes/log/components/log-elasticsearch/resources/config/elasticsearch.yml b/kubernetes/log/components/log-elasticsearch/resources/config/elasticsearch.yml new file mode 100644 index 0000000000..d39fc97e0d --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/resources/config/elasticsearch.yml @@ -0,0 +1,146 @@ +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Name of the Elasticsearch cluster. +# A node can only join a cluster when it shares its cluster.name with all the other nodes in the cluster. +# The default name is elasticsearch, but you should change it to an appropriate name which describes the +# purpose of the cluster. +# +cluster.name: "onap-log" +# +# The port that other nodes in the cluster should use when communicating with this node. +# Required for Elasticsearch's nodes running on different cluster nodes. +# More : https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html +#transport.publish_port:$transport.publish_port +# +# The host address to publish for nodes in the cluster to connect to. +# Required for Elasticsearch's nodes running on different cluster nodes. +# More : https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html +#transport.publish_host:$transport.publish_host +# +# ------------------------------------ Node ------------------------------------ +# +# It is better to provide different meaningfull names fot different elastic nodes. +# By default, Elasticsearch will take the 7 first character of the randomly generated uuid used as the node id. +# Note that the node id is persisted and does not change when a node restarts +# +#node.name: $node.name +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# The location of the data files of each index / shard allocated on the node. Can hold multiple locations separated by coma. +# In production, we should not keep this default to "/elasticsearch/data", as on upgrading Elasticsearch, directory structure +# may change & can deal to data loss. +path.data: /usr/share/elasticsearch/data +# +# Elasticsearch's log files location. In production, we should not keep this default to "/elasticsearch/logs", +# as on upgrading Elasticsearch, directory structure may change. +path.logs: /usr/share/elasticsearch/logs +# +# ----------------------------------- Memory ----------------------------------- +# +# It is vitally important to the health of your node that none of the JVM is ever swapped out to disk. +# Lock the memory on startup. +# +bootstrap.memory_lock: false +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# In order to communicate and to form a cluster with nodes on other servers, your node will need to bind to a +# non-loopback address. +network.host: 0.0.0.0 +# +# Set a custom port for HTTP: If required, default is 9200-9300 +# +#http.port: $http.port +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started +# To form a cluster with nodes on other servers, you have to provide a seed list of other nodes in the cluster +# that are likely to be live and contactable. +# By default, Elasticsearch will bind to the available loopback addresses and will scan ports 9300 to 9305 to try +# to connect to other nodes running on the same server. +# +#$discovery.zen.ping.unicast.hosts +# +# This setting tells Elasticsearch to not elect a master unless there are enough master-eligible nodes +# available. Only then will an election take place. +# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): +discovery.zen.minimum_master_nodes: 1 +# +# For more information, consult the zen discovery module documentation. +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +#gateway.recover_after_nodes: 3 +# +# For more information, consult the gateway module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Require explicit names when deleting indices: +# +#action.destructive_requires_name: true +# Set a custom port for HTTP: If required, default is 9200-9300 +# This is used for REST APIs +http.port: {{.Values.service.externalPort}} +# Port to bind for communication between nodes. Accepts a single value or a range. +# If a range is specified, the node will bind to the first available port in the range. +# Defaults to 9300-9400. +# More info: +transport.tcp.port: {{.Values.service.externalPort2}} + +xpack.graph.enabled: false +#Set to false to disable X-Pack graph features. + +xpack.ml.enabled: false +#Set to false to disable X-Pack machine learning features. + +xpack.monitoring.enabled: false +#Set to false to disable X-Pack monitoring features. + +xpack.security.enabled: false +#Set to false to disable X-Pack security features. + +xpack.watcher.enabled: false +#Set to false to disable Watcher. diff --git a/kubernetes/log/components/log-elasticsearch/templates/NOTES.txt b/kubernetes/log/components/log-elasticsearch/templates/NOTES.txt new file mode 100644 index 0000000000..ab908cd309 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/NOTES.txt @@ -0,0 +1,33 @@ +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.externalPort }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} +{{- end }} diff --git a/kubernetes/log/components/log-elasticsearch/templates/configmap.yaml b/kubernetes/log/components/log-elasticsearch/templates/configmap.yaml new file mode 100644 index 0000000000..fe0349ede9 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/configmap.yaml @@ -0,0 +1,29 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-configmap + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/components/log-elasticsearch/templates/deployment.yaml b/kubernetes/log/components/log-elasticsearch/templates/deployment.yaml new file mode 100644 index 0000000000..6a0e6d2e3d --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/deployment.yaml @@ -0,0 +1,112 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} + spec: + initContainers: + - command: + - /bin/sh + - -c + - | + sysctl -w vm.max_map_count=262144 + mkdir -p /logroot/elasticsearch/logs + mkdir -p /logroot/elasticsearch/data + chmod -R 777 /logroot/elasticsearch + chown -R root:root /logroot + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + securityContext: + privileged: true + image: {{ include "repositoryGenerator.image.busybox" . }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: init-sysctl + volumeMounts: + - name: {{ include "common.fullname" . }}-logs + mountPath: /logroot/ + containers: + - name: {{ include "common.name" . }} + image: {{ include "repositoryGenerator.elasticRepository" . }}/{{ .Values.image }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + resources: +{{ include "common.resources" . | indent 12 }} + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + - containerPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} +# disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end -}} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort2 }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + env: + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + name: {{ include "common.fullname" . }}-config + subPath: elasticsearch.yml + - mountPath: /usr/share/elasticsearch/data/ + name: {{ include "common.fullname" . }}-data + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }}-config + configMap: + name: {{ include "common.fullname" . }}-configmap + items: + - key: elasticsearch.yml + path: elasticsearch.yml + - name: {{ include "common.fullname" . }}-data + persistentVolumeClaim: + claimName: {{ include "common.fullname" . }} + - name: {{ include "common.fullname" . }}-logs + hostPath: + path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ include "common.release" . }}/{{ .Values.persistence.mountSubPathLogs }} + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/components/log-elasticsearch/templates/ingress.yaml b/kubernetes/log/components/log-elasticsearch/templates/ingress.yaml new file mode 100644 index 0000000000..8f87c68f1e --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/ingress.yaml @@ -0,0 +1 @@ +{{ include "common.ingress" . }} diff --git a/kubernetes/log/components/log-elasticsearch/templates/pv.yaml b/kubernetes/log/components/log-elasticsearch/templates/pv.yaml new file mode 100644 index 0000000000..9d4093db11 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/pv.yaml @@ -0,0 +1,37 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +kind: PersistentVolume +apiVersion: v1 +metadata: + name: {{ include "common.fullname" . }}-data + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" + release: "{{ include "common.release" . }}" + heritage: "{{ .Release.Service }}" + name: {{ include "common.fullname" . }} +spec: + capacity: + storage: {{ .Values.persistence.size}} + accessModes: + - {{ .Values.persistence.accessMode }} + persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }} + storageClassName: "{{ include "common.fullname" . }}-data" + hostPath: + path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ include "common.release" . }}/{{ .Values.persistence.mountSubPath }} +{{- end -}} diff --git a/kubernetes/log/components/log-elasticsearch/templates/pvc.yaml b/kubernetes/log/components/log-elasticsearch/templates/pvc.yaml new file mode 100644 index 0000000000..6ae4eea0d3 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/pvc.yaml @@ -0,0 +1,38 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ include "common.release" . }}" + heritage: "{{ .Release.Service }}" +{{- if .Values.persistence.annotations }} + annotations: +{{ toYaml .Values.persistence.annotations | indent 4 }} +{{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + storageClassName: {{ include "common.storageClass" . }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end -}} diff --git a/kubernetes/log/components/log-elasticsearch/templates/service.yaml b/kubernetes/log/components/log-elasticsearch/templates/service.yaml new file mode 100644 index 0000000000..7736f0c9d7 --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/templates/service.yaml @@ -0,0 +1,70 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.internalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.service.name }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name2 }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type2 }} + ports: + {{if eq .Values.service.type2 "NodePort" -}} + - port: {{ .Values.service.internalPort2 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} + name: {{ .Values.service.name2 }} + {{- else -}} + - port: {{ .Values.service.externalPort2 }} + targetPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} diff --git a/kubernetes/log/components/log-elasticsearch/values.yaml b/kubernetes/log/components/log-elasticsearch/values.yaml new file mode 100644 index 0000000000..7919a0948e --- /dev/null +++ b/kubernetes/log/components/log-elasticsearch/values.yaml @@ -0,0 +1,116 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + persistence: {} + +################################################################# +# Application configuration defaults. +################################################################# + +# application image +image: elasticsearch/elasticsearch:5.5.0 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +# Example: +config: {} + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 120 + periodSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 120 + periodSeconds: 10 + +## Persist data to a persitent volume +persistence: + enabled: true + + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + # existingClaim: + volumeReclaimPolicy: Retain + + ## database data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + accessMode: ReadWriteOnce + size: 1Gi + mountPath: /dockerdata-nfs + mountSubPath: log/elasticsearch/data + mountSubPathLogs: log + +service: + type: NodePort + name: log-es + externalPort: 9200 + internalPort: 9200 + nodePort: 54 + type2: ClusterIP + name2: log-es-tcp + externalPort2: 9300 + internalPort2: 9300 + +ingress: + enabled: false + service: + - baseaddr: "loges" + name: "log-es" + port: 9200 + config: + ssl: "none" +# Resource Limit flavor -By Default using small +flavor: small +# Segregation for Different environment (Small and Large) +resources: + small: + limits: + cpu: 1 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi + large: + limits: + cpu: 2 + memory: 8Gi + requests: + cpu: 1 + memory: 4Gi + unlimited: {} diff --git a/kubernetes/log/components/log-kibana/Chart.yaml b/kubernetes/log/components/log-kibana/Chart.yaml new file mode 100644 index 0000000000..8f5d973177 --- /dev/null +++ b/kubernetes/log/components/log-kibana/Chart.yaml @@ -0,0 +1,19 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: ONAP Logging Kibana +name: log-kibana +version: 6.0.0 diff --git a/kubernetes/log/components/log-kibana/requirements.yaml b/kubernetes/log/components/log-kibana/requirements.yaml new file mode 100644 index 0000000000..bdc19209e7 --- /dev/null +++ b/kubernetes/log/components/log-kibana/requirements.yaml @@ -0,0 +1,22 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~6.x-0 + repository: '@local' + - name: repositoryGenerator + version: ~6.x-0 + repository: '@local' diff --git a/kubernetes/log/components/log-kibana/resources/config/README.txt b/kubernetes/log/components/log-kibana/resources/config/README.txt new file mode 100644 index 0000000000..2863c1d5e5 --- /dev/null +++ b/kubernetes/log/components/log-kibana/resources/config/README.txt @@ -0,0 +1 @@ +"kibana-onboarding.json" file contains initial setup of Kibana obtained using Elasticdump tool. \ No newline at end of file diff --git a/kubernetes/log/components/log-kibana/resources/config/kibana-onboarding.json b/kubernetes/log/components/log-kibana/resources/config/kibana-onboarding.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kubernetes/log/components/log-kibana/resources/config/kibana.yml b/kubernetes/log/components/log-kibana/resources/config/kibana.yml new file mode 100644 index 0000000000..377f3c7b65 --- /dev/null +++ b/kubernetes/log/components/log-kibana/resources/config/kibana.yml @@ -0,0 +1,129 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +xpack.graph.enabled: false +#Set to false to disable X-Pack graph features. +xpack.ml.enabled: false +#Set to false to disable X-Pack machine learning features. +xpack.monitoring.enabled: false +#Set to false to disable X-Pack monitoring features. +xpack.reporting.enabled: false +#Set to false to disable X-Pack reporting features. +xpack.security.enabled: false +#Set to false to disable X-Pack security features. +xpack.watcher.enabled: false +#Set to false to disable Watcher. +# Kibana is served by a back end server. This setting specifies the port to use. +server.port: {{.Values.service.externalPort}} + +# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values. +# The default is 'localhost', which usually means remote machines will not be able to connect. +# To allow connections from remote users, set this parameter to a non-loopback address. +server.host: "0" + +# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects +# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests +# to Kibana. This setting cannot end in a slash. +#server.basePath: "" + +# The maximum payload size in bytes for incoming server requests. +#server.maxPayloadBytes: 1048576 + +# The Kibana server's name. This is used for display purposes. +server.name: "Kibana" + +# The URL of the Elasticsearch instance to use for all your queries. +elasticsearch.url: "http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}" +# When this setting's value is true Kibana uses the hostname specified in the server.host +# setting. When the value of this setting is false, Kibana uses the hostname of the host +# that connects to this Kibana instance. +#elasticsearch.preserveHost: true + +# Kibana uses an index in Elasticsearch to store saved searches, visualizations and +# dashboards. Kibana creates a new index if the index doesn't already exist. +#kibana.index: ".kibana" + +# The default application to load. +#kibana.defaultAppId: "discover" + +# If your Elasticsearch is protected with basic authentication, these settings provide +# the username and password that the Kibana server uses to perform maintenance on the Kibana +# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which +# is proxied through the Kibana server. +elasticsearch.username: "elastic" +elasticsearch.password: "changeme" +# Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively. +# These settings enable SSL for outgoing requests from the Kibana server to the browser. +#server.ssl.enabled: $server_ssl_enabled +#server.ssl.certificate: $server_ssl_certificate +#server.ssl.key: $server_ssl_key + +# Optional settings that provide the paths to the PEM-format SSL certificate and key files. +# These files validate that your Elasticsearch backend uses the same key files. +#elasticsearch.ssl.certificate: $elasticsearch_ssl_certificate +#elasticsearch.ssl.key: $elasticsearch_ssl_key + +# Optional setting that enables you to specify a path to the PEM file for the certificate +# authority for your Elasticsearch instance. +#elasticsearch.ssl.certificateAuthorities: $elasticsearch_ssl_certificateAuthorities + +# To disregard the validity of SSL certificates, change this setting's value to 'none'. +#elasticsearch.ssl.verificationMode: $elasticsearch_ssl_verificationMode + +# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of +# the elasticsearch.requestTimeout setting. +#elasticsearch.pingTimeout: 1500 + +# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value +# must be a positive integer. +#elasticsearch.requestTimeout: 30000 + +# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side +# headers, set this value to [] (an empty list). +#elasticsearch.requestHeadersWhitelist: [ authorization ] + +# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten +# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration. +#elasticsearch.customHeaders: {} + +# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable. +#elasticsearch.shardTimeout: 0 + +# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying. +#elasticsearch.startupTimeout: 5000 + +# Specifies the path where Kibana creates the process ID file. +#pid.file: /var/run/kibana.pid + +# Enables you specify a file where Kibana stores log output. +#logging.dest: stdout + +# Set the value of this setting to true to suppress all logging output. +#logging.silent: false + +# Set the value of this setting to true to suppress all logging output other than error messages. +#logging.quiet: false + +# Set the value of this setting to true to log all events, including system usage information +# and all requests. +#logging.verbose: false + +# Set the interval in milliseconds to sample system and process performance +# metrics. Minimum is 100ms. Defaults to 5000. +#ops.interval: 5000 + +# The default locale. This locale can be used in certain circumstances to substitute any missing +# translations. +#i18n.defaultLocale: "en" diff --git a/kubernetes/log/components/log-kibana/templates/NOTES.txt b/kubernetes/log/components/log-kibana/templates/NOTES.txt new file mode 100644 index 0000000000..f115eb6f23 --- /dev/null +++ b/kubernetes/log/components/log-kibana/templates/NOTES.txt @@ -0,0 +1,32 @@ +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.externalPort }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} +{{- end }} diff --git a/kubernetes/log/components/log-kibana/templates/configmap.yaml b/kubernetes/log/components/log-kibana/templates/configmap.yaml new file mode 100644 index 0000000000..3e98246df1 --- /dev/null +++ b/kubernetes/log/components/log-kibana/templates/configmap.yaml @@ -0,0 +1,29 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/components/log-kibana/templates/deployment.yaml b/kubernetes/log/components/log-kibana/templates/deployment.yaml new file mode 100644 index 0000000000..a1824d2509 --- /dev/null +++ b/kubernetes/log/components/log-kibana/templates/deployment.yaml @@ -0,0 +1,114 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} + spec: + initContainers: + - command: + - /app/ready.py + args: + - --container-name + - log-elasticsearch + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ include "repositoryGenerator.image.readiness" . }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-readiness + - args: + - --input=/config/kibana-onboarding.json + - --output=http://{{.Values.config.elasticsearchServiceName}}.{{ include "common.namespace" . }}:{{.Values.config.elasticsearchPort}}/.kibana + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.elasticdumpImage }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-elasticdump + volumeMounts: + - mountPath: /config/kibana-onboarding.json + name: {{ include "common.fullname" . }} + subPath: kibana-onboarding.json + containers: + - name: {{ include "common.name" . }} + image: {{ include "repositoryGenerator.elasticRepository" . }}/{{ .Values.image }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + resources: +{{ include "common.resources" . | indent 12 }} + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + readinessProbe: + httpGet: + path: "/" + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} +# disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + httpGet: + path: "/" + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + {{ end -}} + env: + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /usr/share/kibana/config/ + name: {{ include "common.fullname" . }} + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }} + configMap: + name: {{ include "common.fullname" . }} + items: + - key: kibana.yml + path: kibana.yml + - key: kibana-onboarding.json + path: kibana-onboarding.json + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/components/log-kibana/templates/ingress.yaml b/kubernetes/log/components/log-kibana/templates/ingress.yaml new file mode 100644 index 0000000000..8f87c68f1e --- /dev/null +++ b/kubernetes/log/components/log-kibana/templates/ingress.yaml @@ -0,0 +1 @@ +{{ include "common.ingress" . }} diff --git a/kubernetes/log/components/log-kibana/templates/service.yaml b/kubernetes/log/components/log-kibana/templates/service.yaml new file mode 100644 index 0000000000..c53dc03368 --- /dev/null +++ b/kubernetes/log/components/log-kibana/templates/service.yaml @@ -0,0 +1,43 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.externalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.service.name }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} diff --git a/kubernetes/log/components/log-kibana/values.yaml b/kubernetes/log/components/log-kibana/values.yaml new file mode 100644 index 0000000000..767ea6ae99 --- /dev/null +++ b/kubernetes/log/components/log-kibana/values.yaml @@ -0,0 +1,100 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + persistence: {} + +################################################################# +# Application configuration defaults. +################################################################# + +# Elasticdump image +elasticdumpImage: taskrabbit/elasticsearch-dump + +# application image +image: kibana/kibana:5.5.0 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +config: + elasticsearchServiceName: log-es + elasticsearchPort: 9200 + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 300 + periodSeconds: 10 + timeoutSeconds: 1 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 300 + periodSeconds: 10 + timeoutSeconds: 1 + +service: + #Example service definition with external, internal and node ports. + #Services may use any combination of ports depending on the 'type' of + #service being defined. + type: NodePort + name: log-kibana + externalPort: 5601 + internalPort: 5601 + nodePort: 53 + +ingress: + enabled: false + service: + - baseaddr: "kibana.api" + name: "log-kibana" + port: 5601 + config: + ssl: "none" + +# Resource Limit flavor -By Default using small +flavor: small +# Segregation for Different environment (Small and Large) +resources: + small: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi + large: + limits: + cpu: 4 + memory: 8Gi + requests: + cpu: 2 + memory: 4Gi + unlimited: {} diff --git a/kubernetes/log/components/log-logstash/Chart.yaml b/kubernetes/log/components/log-logstash/Chart.yaml new file mode 100644 index 0000000000..8349548f86 --- /dev/null +++ b/kubernetes/log/components/log-logstash/Chart.yaml @@ -0,0 +1,19 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: ONAP Logging Logstash +name: log-logstash +version: 6.0.0 diff --git a/kubernetes/log/components/log-logstash/requirements.yaml b/kubernetes/log/components/log-logstash/requirements.yaml new file mode 100644 index 0000000000..bdc19209e7 --- /dev/null +++ b/kubernetes/log/components/log-logstash/requirements.yaml @@ -0,0 +1,22 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~6.x-0 + repository: '@local' + - name: repositoryGenerator + version: ~6.x-0 + repository: '@local' diff --git a/kubernetes/log/components/log-logstash/resources/config/logstash.yml b/kubernetes/log/components/log-logstash/resources/config/logstash.yml new file mode 100644 index 0000000000..7c3bd8f851 --- /dev/null +++ b/kubernetes/log/components/log-logstash/resources/config/logstash.yml @@ -0,0 +1,32 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +http.host: "0.0.0.0" +pipeline.workers: 3 +## Path where pipeline configurations reside +path.config: /usr/share/logstash/pipeline + +## Type of queue : memeory based or file based +#queue.type: persisted +## Size of queue +#queue.max_bytes: 1024mb +## Setting true makes logstash check periodically for change in pipeline configurations +config.reload.automatic: true + +## xpack configurations +#xpack.monitoring.elasticsearch.url: ["http://10.247.186.12:9200", "http://10.247.186.13:9200"] +#xpack.monitoring.elasticsearch.username: elastic +#xpack.monitoring.elasticsearch.password: changeme +xpack.monitoring.enabled: false diff --git a/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf b/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf new file mode 100644 index 0000000000..d6b0696b81 --- /dev/null +++ b/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf @@ -0,0 +1,276 @@ +{{/* +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} +input { + beats { + + ## Add a id to plugin configuration. Can be anything unique. + id => 'beats_plugin' + + ######## Connection configurations ######## + + ## The port to listen on. + port => {{.Values.service.externalPort}} + + ## Close Idle clients after the specified time in seconds. Default is 60 seconds + #client_inactivity_timeout => 60 + + ######## Security configurations ######## + + ## Enable encryption. Default false. + #ssl => $filebeat_ssl + + ## ssl certificate path. + #ssl_certificate => $filebeat_ssl_certificate + + ## SSL key to use. + #ssl_key => $filebeat_ssl_key + + ##SSL key passphrase to use. + #ssl_key_passphrase => $filebeat_ssl_key_passphrase + + ## Value can be any of: none, peer, force_peer. + #ssl_verify_mode => $filebeat_ssl_verify_mode + + ## Time in milliseconds for an incomplete ssl handshake to timeout. Default is 10000 ms. + #ssl_handshake_timeout => 10000 + include_codec_tag => false + } +} + + +filter { + grok { + break_on_match => false + match => { + "source" => ["/var/log/onap/(?[^/]+)/", + "/var/log/onap/%{GREEDYDATA:componentLogFile}" + ] + } + } + + # Filter for log4j xml events + if "" in [message] { + + #mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug + + #Filter to parse xml event and retrieve data + xml { + source => "message" + store_xml => false + remove_namespaces => true + target => "xml_content" + xpath => [ "/event/message/text()", "logmsg" , + "/event/@logger", "Logger", + "/event/@timestamp", "Timestamp", + "/event/@level", "loglevel", + "/event/@thread", "Thread", + "/event/throwable/text()", "Exceptionthrowable", + "/event/NDC/text()", "NDCs", + "/event/properties/data/@name","mdcname", + "/event/properties/data/@value","mdcvalue"] + + } + + #Ruby filter to iterate and separate MDCs into documents + ruby { + code => ' + $i = 0 + $num = 0 + if event.get("[mdcname]") + $num = event.get("[mdcname]").length + end + if $num != 0 + until $i > $num do + if event.get("[mdcname]").at($i) and event.get("[mdcvalue]").at($i) + event.set(event.get("[mdcname]").at($i), event.get("[mdcvalue]").at($i)) + end + $i=$i+1 + end + end + ' + } + + #Validations + if [Exceptionthrowable] + { + mutate { + replace => { + "exceptionmessage" => "%{[Exceptionthrowable]}" + } + } + } + + if [NDCs] + { + mutate { + replace => { + "NDC" => "%{[NDCs]}" + } + } + } + + mutate { + replace => { + "Logger" =>"%{[Logger]}" + "logmsg" =>"%{[logmsg]}" + "Timestamp" =>"%{[Timestamp]}" + "loglevel" =>"%{[loglevel]}" + "message" => "%{logmsg}" + "Thread" => "%{[Thread]}" + } + remove_field => ["mdcname", "mdcvalue", "logmsg","Exceptionthrowable","NDCs"] + } + + if [Timestamp] + { + date { + match => ["Timestamp", "UNIX_MS"] + target => "Timestamp" + } + } + } + # Filter for logback events + else { + + #mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug + + mutate { + gsub => [ + 'message', ' = ', '=', + 'message', '= ', '=null', + 'message', '=\t', '=null ', #This null is followed by a tab + 'message', '\t$', '\t' + ] + } + # The grok below parses the message field for all current logback patterns used by oom components. + # Example logback pattern: %d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg + # Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message} + # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq + grok { + match => { + "message" => [ + "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}", + "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}" + ] + } + overwrite => ["message"] + } + # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values. + kv { + source => "MDCs" + field_split => ",\t" + trim_key => "\s" + trim_value => "\s" + remove_field => [ "MDCs" ] + } + + if (![Timestamp] and [EndTimestamp]) { + mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } } + } + date { + match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ] + target => "Timestamp" + } + + mutate { + remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"] + } + + if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") { + #Parse kvps in message + kv { + field_split => "\s" + trim_key => "\s" + trim_value => "\s" + } + + #If Request Id is missing and DID is present use as RequestId + if (![RequestId] and [DID] =~ /.+/) { + mutate { add_field => { "RequestId" => "%{DID}" } } + } + } + + } #Close else statement for logback events +} #Close filter + + +output { + elasticsearch { + id => 'onap_es' + + ######### Security configurations ######### + + user => "elastic" + password => "changeme" + + ## The .cer or .pem file to validate the server's certificate + #cacert => $es_cacert + + ## The keystore used to present a certificate to the server. It can be either .jks or .p12 + #keystore => $es_keystore + #keystore_password => $es_keystore_password + + ## Enable SSL/TLS secured communication to Elasticsearch cluster. + ## Default is not set which in that case depends on the protocol specidfied in hosts list + #ssl => $es_ssl + + ## Option to validate the server's certificate. Default is true + #ssl_certificate_verification => $es_ssl_certificate_verification + + ## The JKS truststore to validate the server's certificate. + #truststore => $es_truststore + #truststore_password => $es_truststore_password + + + ######### Elasticsearchcluster and host configurations ######### + + ##can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered + hosts => ["http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}"] + + + ## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false. + sniffing => true + + ## How long to wait, in seconds, between sniffing attempts. Default is 5 seconds. + #sniffing_delay => 5 + + ## Set the address of a forward HTTP proxy. + #proxy => $es_proxy + + ##Use this if you must run Elasticsearch behind a proxy that remaps the root path for the Elasticsearch HTTP API lives + #path => $es_path + + ######### Elasticsearch request configurations ######### + + ## This setting defines the maximum sized bulk request Logstash will make. + #flush_size => ? + + ######### Document configurations ######### + + index => "logstash-%{+YYYY.MM.dd}" + document_type => "logs" + + ## This can be used to associate child documents with a parent using the parent ID. + #parent => "abcd' + } +} + diff --git a/kubernetes/log/components/log-logstash/templates/NOTES.txt b/kubernetes/log/components/log-logstash/templates/NOTES.txt new file mode 100644 index 0000000000..f115eb6f23 --- /dev/null +++ b/kubernetes/log/components/log-logstash/templates/NOTES.txt @@ -0,0 +1,32 @@ +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ include "common.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "common.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ include "common.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "common.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ include "common.namespace" . }} {{ include "common.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.externalPort }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ include "common.namespace" . }} -l "app={{ include "common.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:{{ .Values.service.internalPort }} +{{- end }} diff --git a/kubernetes/log/components/log-logstash/templates/configmap.yaml b/kubernetes/log/components/log-logstash/templates/configmap.yaml new file mode 100644 index 0000000000..3e98246df1 --- /dev/null +++ b/kubernetes/log/components/log-logstash/templates/configmap.yaml @@ -0,0 +1,29 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/log/components/log-logstash/templates/deployment.yaml b/kubernetes/log/components/log-logstash/templates/deployment.yaml new file mode 100644 index 0000000000..566c7a3b10 --- /dev/null +++ b/kubernetes/log/components/log-logstash/templates/deployment.yaml @@ -0,0 +1,102 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} + spec: + initContainers: + - command: + - /app/ready.py + args: + - --container-name + - log-elasticsearch + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ include "repositoryGenerator.image.readiness" . }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-readiness + containers: + - name: {{ include "common.name" . }} + image: {{ include "repositoryGenerator.elasticRepository" . }}/{{ .Values.image }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + resources: +{{ include "common.resources" . | indent 12 }} + ports: + - containerPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + - containerPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} +# disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end -}} + env: + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /usr/share/logstash/config/ + name: {{ include "common.fullname" . }}-config + - mountPath: /usr/share/logstash/pipeline/ + name: {{ include "common.fullname" . }}-pipeline + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }}-config + configMap: + name: {{ include "common.fullname" . }} + items: + - key: logstash.yml + path: logstash.yml + - name: {{ include "common.fullname" . }}-pipeline + configMap: + name: {{ include "common.fullname" . }} + items: + - key: onap-pipeline.conf + path: onap-pipeline.conf + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/log/components/log-logstash/templates/ingress.yaml b/kubernetes/log/components/log-logstash/templates/ingress.yaml new file mode 100644 index 0000000000..8f87c68f1e --- /dev/null +++ b/kubernetes/log/components/log-logstash/templates/ingress.yaml @@ -0,0 +1 @@ +{{ include "common.ingress" . }} diff --git a/kubernetes/log/components/log-logstash/templates/service.yaml b/kubernetes/log/components/log-logstash/templates/service.yaml new file mode 100644 index 0000000000..7736f0c9d7 --- /dev/null +++ b/kubernetes/log/components/log-logstash/templates/service.yaml @@ -0,0 +1,70 @@ +{{/* +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.internalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.service.name }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.name }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name2 }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type2 }} + ports: + {{if eq .Values.service.type2 "NodePort" -}} + - port: {{ .Values.service.internalPort2 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} + name: {{ .Values.service.name2 }} + {{- else -}} + - port: {{ .Values.service.externalPort2 }} + targetPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} diff --git a/kubernetes/log/components/log-logstash/values.yaml b/kubernetes/log/components/log-logstash/values.yaml new file mode 100644 index 0000000000..7a0674cdf5 --- /dev/null +++ b/kubernetes/log/components/log-logstash/values.yaml @@ -0,0 +1,102 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + persistence: {} + +################################################################# +# Application configuration defaults. +################################################################# + +# application image +image: logstash/logstash:5.4.3 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +config: + elasticsearchServiceName: log-es + elasticsearchPort: 9200 + +# default number of instances +# 30+ logs/sec will saturate a single node to 6+ vCores +replicaCount: 3 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 120 + periodSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 10 + periodSeconds: 10 + +service: + #Example service definition with external, internal and node ports. + #Services may use any combination of ports depending on the 'type' of + #service being defined. + type: NodePort + name: log-ls + externalPort: 5044 + internalPort: 5044 + nodePort: 55 + type2: ClusterIP + name2: log-ls-http + externalPort2: 9600 + internalPort2: 9600 +ingress: + enabled: false + service: + - baseaddr: "logls" + name: "log-ls" + port: 5044 + - baseaddr: "loglshttp" + name: "log-ls" + port: 9600 + config: + ssl: "none" + +# Resource Limit flavor -By Default using small +flavor: small +# Segregation for Different environment (Small and Large) +resources: + small: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 0.5 + memory: 1Gi + large: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 2 + memory: 4Gi + unlimited: {} diff --git a/kubernetes/log/requirements.yaml b/kubernetes/log/requirements.yaml index a7089ea6b3..ae81fc8441 100644 --- a/kubernetes/log/requirements.yaml +++ b/kubernetes/log/requirements.yaml @@ -16,3 +16,16 @@ dependencies: - name: common version: ~6.x-0 repository: '@local' + - name: repositoryGenerator + version: ~6.x-0 + repository: '@local' + - name: log-elasticsearch + version: ~6.x-0 + repository: 'file://components/log-elasticsearch' + - name: log-kibana + version: ~6.x-0 + repository: 'file://components/log-kibana' + - name: log-logstash + version: ~6.x-0 + repository: 'file://components/log-logstash' + diff --git a/kubernetes/log/values.yaml b/kubernetes/log/values.yaml index bb41a8717b..ddcf5235cd 100644 --- a/kubernetes/log/values.yaml +++ b/kubernetes/log/values.yaml @@ -18,4 +18,3 @@ ################################################################# global: nodePortPrefix: 302 - readinessImage: onap/oom/readiness:3.0.1 -- cgit 1.2.3-korg