From 417b3b46c64f965bed72793b3572ee89b73b22e6 Mon Sep 17 00:00:00 2001 From: Geora Barsky Date: Fri, 17 Aug 2018 13:31:36 -0400 Subject: Add Search Data Service to POMBA OOM Issue-ID: LOG-612 Change-Id: I3ef3707e332c44d3dbaee5a9601607063b99c3ea Signed-off-by: Geora Barsky --- .../resources/config/elasticsearch.yml | 2 +- .../pomba/charts/pomba-search-data/.helmignore | 21 +++ .../pomba/charts/pomba-search-data/Chart.yaml | 18 +++ .../resources/config/analysis-config.json | 32 ++++ .../resources/config/auth/search_policy.json | 31 ++++ .../resources/config/auth/tomcat_keystore | Bin 0 -> 2214 bytes .../resources/config/dynamic-custom-template.json | 12 ++ .../resources/config/elastic-search.properties | 5 + .../resources/config/es-payload-translation.json | 20 +++ .../resources/config/filter-config.json | 7 + .../resources/config/log/logback.xml | 177 +++++++++++++++++++++ .../pomba-search-data/templates/configmap.yaml | 15 ++ .../pomba-search-data/templates/deployment.yaml | 143 +++++++++++++++++ .../charts/pomba-search-data/templates/secret.yaml | 16 ++ .../pomba-search-data/templates/service.yaml | 24 +++ .../pomba/charts/pomba-search-data/values.yaml | 67 ++++++++ 16 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 kubernetes/pomba/charts/pomba-search-data/.helmignore create mode 100644 kubernetes/pomba/charts/pomba-search-data/Chart.yaml create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/analysis-config.json create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/auth/search_policy.json create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/auth/tomcat_keystore create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/dynamic-custom-template.json create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/elastic-search.properties create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/es-payload-translation.json create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/filter-config.json create mode 100644 kubernetes/pomba/charts/pomba-search-data/resources/config/log/logback.xml create mode 100644 kubernetes/pomba/charts/pomba-search-data/templates/configmap.yaml create mode 100644 kubernetes/pomba/charts/pomba-search-data/templates/deployment.yaml create mode 100644 kubernetes/pomba/charts/pomba-search-data/templates/secret.yaml create mode 100644 kubernetes/pomba/charts/pomba-search-data/templates/service.yaml create mode 100644 kubernetes/pomba/charts/pomba-search-data/values.yaml diff --git a/kubernetes/pomba/charts/pomba-elasticsearch/resources/config/elasticsearch.yml b/kubernetes/pomba/charts/pomba-elasticsearch/resources/config/elasticsearch.yml index 80b68b97ab..2e469d551f 100644 --- a/kubernetes/pomba/charts/pomba-elasticsearch/resources/config/elasticsearch.yml +++ b/kubernetes/pomba/charts/pomba-elasticsearch/resources/config/elasticsearch.yml @@ -17,7 +17,7 @@ # The default name is elasticsearch, but you should change it to an appropriate name which describes the # purpose of the cluster. # -cluster.name: "pomba-dashboard" +cluster.name: POMBA_ES # # The port that other nodes in the cluster should use when communicating with this node. # Required for Elasticsearch's nodes running on different cluster nodes. diff --git a/kubernetes/pomba/charts/pomba-search-data/.helmignore b/kubernetes/pomba/charts/pomba-search-data/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/kubernetes/pomba/charts/pomba-search-data/Chart.yaml b/kubernetes/pomba/charts/pomba-search-data/Chart.yaml new file mode 100644 index 0000000000..89e7917206 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +apiVersion: v1 +description: ONAP POMBA search-data +name: pomba-search-data +version: 2.0.0 diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/analysis-config.json b/kubernetes/pomba/charts/pomba-search-data/resources/config/analysis-config.json new file mode 100644 index 0000000000..dce0e40b98 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/analysis-config.json @@ -0,0 +1,32 @@ +[ + { + "name": "whitespace_analyzer", + "description": "A standard whitespace analyzer.", + "behaviours": [ + "Tokenize the text using white space characters as delimeters.", + "Convert all characters to lower case.", + "Convert all alphanumeric and symbolic Unicode characters above the first 127 ASCII characters into their ASCII equivalents." + ], + "tokenizer": "whitespace", + "filters": [ + "lowercase", + "asciifolding" + ] + }, + { + "name": "ngram_analyzer", + "description": "An analyzer which performs ngram filtering on the data stream.", + "behaviours": [ + "Tokenize the text using white space characters as delimeters.", + "Convert all characters to lower case.", + "Convert all alphanumeric and symbolic Unicode characters above the first 127 ASCII characters into their ASCII equivalents.", + "Apply ngram filtering using the following values for minimum and maximum size in codepoints of a single n-gram: minimum = 1, maximum = 2." + ], + "tokenizer": "whitespace", + "filters": [ + "lowercase", + "asciifolding", + "ngram_filter" + ] + } +] \ No newline at end of file diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/search_policy.json b/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/search_policy.json new file mode 100644 index 0000000000..a83422753d --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/search_policy.json @@ -0,0 +1,31 @@ +{ + "roles": [ + { + "name": "admin", + "functions": [ + { + "name": "search", + "methods": [ + { + "name": "GET" + }, + { + "name": "DELETE" + }, + { + "name": "PUT" + }, + { + "name": "POST" + } + ] + } + ], + "users": [ + { + "username": "CN=ONAP, OU=ONAP, O=ONAP, L=Ottawa, ST=Ontario, C=CA" + } + ] + } + ] +} \ No newline at end of file diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/tomcat_keystore b/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/tomcat_keystore new file mode 100644 index 0000000000..9eec841aa2 Binary files /dev/null and b/kubernetes/pomba/charts/pomba-search-data/resources/config/auth/tomcat_keystore differ diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/dynamic-custom-template.json b/kubernetes/pomba/charts/pomba-search-data/resources/config/dynamic-custom-template.json new file mode 100644 index 0000000000..b0eb539443 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/dynamic-custom-template.json @@ -0,0 +1,12 @@ +"dynamic_templates":[ + { + "strings":{ + "match_mapping_type":"string", + "match": "*", + "mapping":{ + "type":"text", + "fielddata":true + } + } + } +], diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/elastic-search.properties b/kubernetes/pomba/charts/pomba-search-data/resources/config/elastic-search.properties new file mode 100644 index 0000000000..b5c7feed94 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/elastic-search.properties @@ -0,0 +1,5 @@ +# ElasticSearch Configuration + +es.cluster-name=POMBA_ES +es.ip-address=pomba-es.{{.Release.Namespace}} +es.http-port={{ .Values.config.elasticsearchHttpPort }} diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/es-payload-translation.json b/kubernetes/pomba/charts/pomba-search-data/resources/config/es-payload-translation.json new file mode 100644 index 0000000000..93888befbc --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/es-payload-translation.json @@ -0,0 +1,20 @@ +{ + "attr-translations": [ + { + "from": "\"type\":\"string\",\"index\":\"analyzed\"", + "to": "\"type\":\"text\",\"index\":\"true\"" + }, + { + "from": "\"type\":\"string\",\"index\":\"not_analyzed\"", + "to": "\"type\":\"keyword\",\"index\":\"true\"" + }, + { + "from": "\"type\":\"string\"", + "to": "\"type\":\"text\"" + }, + { + "from": "searchable", + "to": "index" + } + ] +} \ No newline at end of file diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/filter-config.json b/kubernetes/pomba/charts/pomba-search-data/resources/config/filter-config.json new file mode 100644 index 0000000000..e2d5285824 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/filter-config.json @@ -0,0 +1,7 @@ +[ + { + "name": "ngram_filter", + "description": "Custom NGram Filter.", + "configuration": " \"type\": \"nGram\", \"min_gram\": 1, \"max_gram\": 50, \"token_chars\": [ \"letter\", \"digit\", \"punctuation\", \"symbol\" ]" + } +] \ No newline at end of file diff --git a/kubernetes/pomba/charts/pomba-search-data/resources/config/log/logback.xml b/kubernetes/pomba/charts/pomba-search-data/resources/config/log/logback.xml new file mode 100644 index 0000000000..3fbbbab864 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/resources/config/log/logback.xml @@ -0,0 +1,177 @@ + + + + + + + + + + + + + + + + + + + + + + + + + ${errorLogPattern} + + + + + + + + + + + ${logDirectory}/${generalLogName}.log + + ${logDirectory}/${generalLogName}.%d{yyyy-MM-dd}.log.zip + + 60 + + + ${errorLogPattern} + + + + + + INFO + + 256 + + + + + + + + ${logDirectory}/${auditLogName}.log + + ${logDirectory}/${auditLogName}.%d{yyyy-MM-dd}.log.zip + + 60 + + + ${auditMetricPattern} + + + + 256 + + + + + ${logDirectory}/${metricsLogName}.log + + ${logDirectory}/${metricsLogName}.%d{yyyy-MM-dd}.log.zip + + 60 + + + + ${auditMetricPattern} + + + + + + 256 + + + + + ${logDirectory}/${debugLogName}.log + + ${logDirectory}/${debugLogName}.%d{yyyy-MM-dd}.log.zip + + 60 + + + ${errorLogPattern} + + + + + 256 + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/kubernetes/pomba/charts/pomba-search-data/templates/configmap.yaml b/kubernetes/pomba/charts/pomba-search-data/templates/configmap.yaml new file mode 100644 index 0000000000..0715f0d51a --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/templates/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-service-log + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/log/logback.xml").AsConfig . | indent 2 }} diff --git a/kubernetes/pomba/charts/pomba-search-data/templates/deployment.yaml b/kubernetes/pomba/charts/pomba-search-data/templates/deployment.yaml new file mode 100644 index 0000000000..c0e2568476 --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/templates/deployment.yaml @@ -0,0 +1,143 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ include "common.name" . }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} + name: {{ include "common.name" . }} + spec: + containers: + - name: {{ include "common.name" . }} + image: "{{ include "common.repository" . }}/{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + env: + - name: CONFIG_HOME + value: /opt/app/search-data-service/config/ + - name: KEY_STORE_PASSWORD + value: {{ .Values.config.keyStorePassword }} + - name: KEY_MANAGER_PASSWORD + value: {{ .Values.config.keyManagerPassword }} + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /opt/app/search-data-service/config/filter-config.json + subPath: filter-config.json + name: {{ include "common.fullname" . }}-service-config + - mountPath: /opt/app/search-data-service/config/elastic-search.properties + subPath: elastic-search.properties + name: {{ include "common.fullname" . }}-service-config + - mountPath: /opt/app/search-data-service/config/analysis-config.json + subPath: analysis-config.json + name: {{ include "common.fullname" . }}-service-config + - mountPath: /opt/app/search-data-service/config/es-payload-translation.json + subPath: es-payload-translation.json + name: {{ include "common.fullname" . }}-service-config + - mountPath: /opt/app/search-data-service/config/dynamic-custom-template.json + subPath: dynamic-custom-template.json + name: {{ include "common.fullname" . }}-service-config + - mountPath: /opt/app/search-data-service/config/auth/tomcat_keystore + subPath: tomcat_keystore + name: {{ include "common.fullname" . }}-service-auth-config + - mountPath: /opt/app/search-data-service/config/auth/search_policy.json + subPath: search_policy.json + name: {{ include "common.fullname" . }}-search-policy-config + - mountPath: /var/log/onap + name: {{ include "common.fullname" . }}-service-logs + - mountPath: /opt/app/search-data-service/bundleconfig/etc/logback.xml + name: {{ include "common.fullname" . }}-service-log-conf + subPath: logback.xml + ports: + - containerPort: {{ .Values.service.internalPort }} + # disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end -}} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + resources: +{{ toYaml .Values.resources | indent 10 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} + {{- end }} + + # side car containers + - name: filebeat-onap + image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + name: filebeat-conf + - mountPath: /var/log/onap + name: {{ include "common.fullname" . }}-service-logs + - mountPath: /usr/share/filebeat/data + name: {{ include "common.fullname" . }}-service-filebeat + + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: {{ include "common.fullname" . }}-service-config + configMap: + name: {{ include "common.fullname" . }} + - name: {{ include "common.fullname" . }}-service-auth-config + secret: + secretName: {{ include "common.fullname" . }}-keystone + - name: {{ include "common.fullname" . }}-search-policy-config + secret: + secretName: {{ include "common.fullname" . }}-policy + - name: filebeat-conf + configMap: + name: {{ .Release.Name }}-pomba-filebeat-configmap + - name: {{ include "common.fullname" . }}-service-logs + emptyDir: {} + - name: {{ include "common.fullname" . }}-service-filebeat + emptyDir: {} + - name: {{ include "common.fullname" . }}-service-log-conf + configMap: + name: {{ include "common.fullname" . }}-service-log + restartPolicy: {{ .Values.global.restartPolicy | default .Values.restartPolicy }} + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/pomba/charts/pomba-search-data/templates/secret.yaml b/kubernetes/pomba/charts/pomba-search-data/templates/secret.yaml new file mode 100644 index 0000000000..33b058fc8f --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/templates/secret.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.fullname" . }}-keystone + namespace: {{ include "common.namespace" . }} +type: Opaque +data: +{{ tpl (.Files.Glob "resources/config/auth/tomcat_keystore").AsSecrets . | indent 2 }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.fullname" . }}-policy + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/auth/search_policy.json").AsSecrets . | indent 2 }} diff --git a/kubernetes/pomba/charts/pomba-search-data/templates/service.yaml b/kubernetes/pomba/charts/pomba-search-data/templates/service.yaml new file mode 100644 index 0000000000..a30bac1afb --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/templates/service.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.servicename" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.internalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.service.portName }} + {{- else -}} + - port: {{ .Values.service.internalPort }} + name: {{ .Values.service.portName }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} diff --git a/kubernetes/pomba/charts/pomba-search-data/values.yaml b/kubernetes/pomba/charts/pomba-search-data/values.yaml new file mode 100644 index 0000000000..c9eb6f94ec --- /dev/null +++ b/kubernetes/pomba/charts/pomba-search-data/values.yaml @@ -0,0 +1,67 @@ +# Default values for search-data. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +global: # global defaults + nodePortPrefix: 302 + readinessRepository: oomk8s + readinessImage: readiness-check:2.0.0 + + +# application image +repository: nexus3.onap.org:10001 +image: onap/search-data-service:1.3-STAGING-latest +pullPolicy: Always +restartPolicy: Always + +# application configuration +config: + elasticsearchHttpPort: 9200 + keyStorePassword: OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 + keyManagerPassword: OBF:1y0q1uvc1uum1uvg1pil1pjl1uuq1uvk1uuu1y10 + + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 10 + periodSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 10 + periodSeconds: 10 + +service: + type: ClusterIP + portName: pomba-search-data + internalPort: 9509 + +ingress: + enabled: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # Example: + # Configure resource requests and limits + # ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # Minimum memory for development is 2 CPU cores and 4GB memory + # Minimum memory for production is 4 CPU cores and 8GB memory +#resources: +# limits: +# cpu: 2 +# memory: 4Gi +# requests: +# cpu: 2 +# memory: 4Gi -- cgit 1.2.3-korg