From 988aeedf9344fefb417af0d8377666b711eb80ce Mon Sep 17 00:00:00 2001 From: Abdelmuhaimen Seaudi Date: Thu, 23 Sep 2021 21:11:44 +0000 Subject: [CPS] Use common postgres for CPS Add option for CPS to use common postgres Issue-ID: OOM-2839 Signed-off-by: Abdelmuhaimen Seaudi Change-Id: Ida133999f26cf50d59103aa30a90c97fba3e66a0 --- kubernetes/common/postgres-init/.helmignore | 21 ++++ kubernetes/common/postgres-init/Chart.yaml | 18 +++ kubernetes/common/postgres-init/requirements.yaml | 21 ++++ .../postgres-init/resources/config/setup.sql | 19 ++++ .../common/postgres-init/templates/configmap.yaml | 29 +++++ kubernetes/common/postgres-init/templates/job.yaml | 121 +++++++++++++++++++++ .../common/postgres-init/templates/secrets.yaml | 16 +++ kubernetes/common/postgres-init/values.yaml | 91 ++++++++++++++++ .../common/postgres/templates/_deployment.tpl | 5 +- .../cps/components/cps-core/requirements.yaml | 6 + .../cps-core/resources/config/application-helm.yml | 5 + kubernetes/cps/components/cps-core/values.yaml | 24 +++- kubernetes/onap/requirements.yaml | 9 ++ kubernetes/onap/resources/overrides/onap-all.yaml | 3 + 14 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 kubernetes/common/postgres-init/.helmignore create mode 100644 kubernetes/common/postgres-init/Chart.yaml create mode 100644 kubernetes/common/postgres-init/requirements.yaml create mode 100644 kubernetes/common/postgres-init/resources/config/setup.sql create mode 100644 kubernetes/common/postgres-init/templates/configmap.yaml create mode 100644 kubernetes/common/postgres-init/templates/job.yaml create mode 100644 kubernetes/common/postgres-init/templates/secrets.yaml create mode 100644 kubernetes/common/postgres-init/values.yaml diff --git a/kubernetes/common/postgres-init/.helmignore b/kubernetes/common/postgres-init/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/kubernetes/common/postgres-init/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/kubernetes/common/postgres-init/Chart.yaml b/kubernetes/common/postgres-init/Chart.yaml new file mode 100644 index 0000000000..7de0d9acb6 --- /dev/null +++ b/kubernetes/common/postgres-init/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright © 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: Chart for Postgres init job +name: postgres-init +version: 8.0.0 diff --git a/kubernetes/common/postgres-init/requirements.yaml b/kubernetes/common/postgres-init/requirements.yaml new file mode 100644 index 0000000000..1a4ab2f2cf --- /dev/null +++ b/kubernetes/common/postgres-init/requirements.yaml @@ -0,0 +1,21 @@ +# Copyright © 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~8.x-0 + repository: 'file://../common' + - name: repositoryGenerator + version: ~8.x-0 + repository: 'file://../repositoryGenerator' diff --git a/kubernetes/common/postgres-init/resources/config/setup.sql b/kubernetes/common/postgres-init/resources/config/setup.sql new file mode 100644 index 0000000000..06e07245be --- /dev/null +++ b/kubernetes/common/postgres-init/resources/config/setup.sql @@ -0,0 +1,19 @@ +--- User Setup +CREATE USER "${PG_USER}" LOGIN; +ALTER USER "${PG_USER}" PASSWORD '${PG_PASSWORD}'; + +CREATE DATABASE ${PG_DATABASE}; +GRANT ALL PRIVILEGES ON DATABASE ${PG_DATABASE} TO "${PG_USER}"; + +--- PG_DATABASE Setup + +\c ${PG_DATABASE} + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +CREATE EXTENSION IF NOT EXISTS pgaudit; + +--- Create schema for PG_USER + +\c ${PG_DATABASE} + +CREATE SCHEMA IF NOT EXISTS "${PG_USER}" AUTHORIZATION "${PG_USER}"; diff --git a/kubernetes/common/postgres-init/templates/configmap.yaml b/kubernetes/common/postgres-init/templates/configmap.yaml new file mode 100644 index 0000000000..66c28a0c69 --- /dev/null +++ b/kubernetes/common/postgres-init/templates/configmap.yaml @@ -0,0 +1,29 @@ +{{/* +# Copyright © 2021 Orange +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/common/postgres-init/templates/job.yaml b/kubernetes/common/postgres-init/templates/job.yaml new file mode 100644 index 0000000000..01151bb4a9 --- /dev/null +++ b/kubernetes/common/postgres-init/templates/job.yaml @@ -0,0 +1,121 @@ +{{/* +# Copyright © 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "common.fullname" . }}-config-job + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ include "common.release" . }} + heritage: {{ .Release.Service }} +spec: + backoffLimit: 20 + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ include "common.release" . }} + name: {{ include "common.name" . }} + spec: + initContainers: + - name: {{ include "common.name" . }}-readiness + command: + - /app/ready.py + args: + - --container-name + - {{ .Values.global.postgres.container.name }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: {{ include "repositoryGenerator.image.readiness" . }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + containers: + - command: + - sh + args: + - -c + - | + function prepare_password { + echo -n $1 | sed -e "s/'/''/g" + } + export PG_PASSWORD=`prepare_password $PG_PASSWORD_INPUT`; + export PG_ROOT_PASSWORD=`prepare_password $PG_ROOT_PASSWORD_INPUT`; + cd /config-input && for PFILE in `ls -1 .`; do envsubst <${PFILE} >/config/${PFILE}; done; + psql "postgresql://postgres:$PG_ROOT_PASSWORD@$PG_HOST" < /config/setup.sql + env: + - name: PG_HOST + value: "{{ .Values.global.postgres.service.name2 }}" + - name: PG_PRIMARY_USER + value: primaryuser + - name: MODE + value: postgres + - name: PG_PRIMARY_PASSWORD_INPUT + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.postgres.secret.primaryPasswordUID" .) "key" "password") | indent 10 }} + - name: PG_USER + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.postgres.secret.userCredentialsUID" .) "key" "login") | indent 10 }} + - name: PG_PASSWORD_INPUT + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.postgres.secret.userCredentialsUID" .) "key" "password") | indent 10 }} + - name: PG_DATABASE + value: "{{ .Values.config.pgDatabase }}" + - name: PG_ROOT_PASSWORD_INPUT + {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.postgres.secret.rootPassUID" .) "key" "password") | indent 10 }} + volumeMounts: + - mountPath: /config-input/setup.sql + name: config + subPath: setup.sql + - mountPath: /config + name: pgconf + image: {{ include "repositoryGenerator.image.postgres" . }} + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }}-update-config + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /config-input/setup.sql + name: config + subPath: setup.sql + - mountPath: /config + name: pgconf + resources: +{{ include "common.resources" . | indent 12 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 10 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 10 }} + {{- end }} + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: config + configMap: + name: {{ include "common.fullname" . }} + - name: pgconf + emptyDir: + medium: Memory + restartPolicy: Never + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/common/postgres-init/templates/secrets.yaml b/kubernetes/common/postgres-init/templates/secrets.yaml new file mode 100644 index 0000000000..f3bea1ff6d --- /dev/null +++ b/kubernetes/common/postgres-init/templates/secrets.yaml @@ -0,0 +1,16 @@ +{{/* +# Copyright © 2021 Orange +# # +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. +*/}} +{{ include "common.secretFast" . }} diff --git a/kubernetes/common/postgres-init/values.yaml b/kubernetes/common/postgres-init/values.yaml new file mode 100644 index 0000000000..7bcd8e23b4 --- /dev/null +++ b/kubernetes/common/postgres-init/values.yaml @@ -0,0 +1,91 @@ +# Copyright © 2021 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + postgres: + service: + name: pgset + container: + name: postgres + +################################################################# +# Secrets metaconfig +################################################################# +secrets: + - uid: '{{ include "common.postgres.secret.rootPassUID" . }}' + type: password + externalSecret: '{{ tpl (default "" .Values.config.pgRootPasswordExternalSecret) . }}' + password: '{{ .Values.config.pgRootPassword }}' + - uid: '{{ include "common.postgres.secret.userCredentialsUID" . }}' + type: basicAuth + externalSecret: '{{ tpl (default "" .Values.config.pgUserExternalSecret) . }}' + login: '{{ .Values.config.pgUserName }}' + password: '{{ .Values.config.pgUserPassword }}' + - uid: '{{ include "common.postgres.secret.primaryPasswordUID" . }}' + type: password + externalSecret: '{{ tpl (default "" .Values.config.pgPrimaryPasswordExternalSecret) . }}' + password: '{{ .Values.config.pgPrimaryPassword }}' + +################################################################# +# Application configuration defaults. +################################################################# + +pullPolicy: Always + +# application configuration +config: + pgUserName: testuser + pgDatabase: userdb + pgDataPath: data + pgRootPasswordExternalSecret: '{{ include "common.namespace" . }}-postgres-db-root-password' + # pgPrimaryPassword: password + # pgUserPassword: password + # pgRootPassword: password + +nodeSelector: {} + +affinity: {} + +flavor: small + +#resources: {} +# We usually recommend not to specify default resources and to leave this as a conscious +# choice for the user. This also increases chances charts run on environments with little +# resources, such as Minikube. If you do want to specify resources, uncomment the following +# lines, adjust them as necessary, and remove the curly braces after 'resources:'. +# +# Example: +# Configure resource requests and limits +# ref: http://kubernetes.io/docs/user-guide/compute-resources/ +# Minimum memory for development is 2 CPU cores and 4GB memory +# Minimum memory for production is 4 CPU cores and 8GB memory +resources: + small: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 10m + memory: 90Mi + large: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 1 + memory: 2Gi + unlimited: {} diff --git a/kubernetes/common/postgres/templates/_deployment.tpl b/kubernetes/common/postgres/templates/_deployment.tpl index d93d401ebc..341b4c86c7 100644 --- a/kubernetes/common/postgres/templates/_deployment.tpl +++ b/kubernetes/common/postgres/templates/_deployment.tpl @@ -1,6 +1,7 @@ {{/* # Copyright © 2018 Amdocs, AT&T, Bell Canada # Copyright © 2020 Samsung Electronics +# Copyright © 2021 Orange # Modifications Copyright (C) 2021 Bell Canada. # # # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -126,9 +127,9 @@ spec: - name: PG_MODE value: {{ $pgMode }} - name: PG_PRIMARY_HOST - value: "{{ $dot.Values.container.name.primary }}" + value: "{{ $dot.Values.service.name2 }}" - name: PG_REPLICA_HOST - value: "{{ $dot.Values.container.name.replica }}" + value: "{{ $dot.Values.service.name3 }}" - name: PG_PRIMARY_PORT value: "{{ $dot.Values.service.internalPort }}" - name: PG_PRIMARY_PASSWORD diff --git a/kubernetes/cps/components/cps-core/requirements.yaml b/kubernetes/cps/components/cps-core/requirements.yaml index d6b6712852..c42e72a232 100644 --- a/kubernetes/cps/components/cps-core/requirements.yaml +++ b/kubernetes/cps/components/cps-core/requirements.yaml @@ -19,6 +19,12 @@ dependencies: - name: postgres version: ~8.x-0 repository: '@local' + condition: global.postgres.localCluster + - name: postgres-init + version: ~8.x-0 + repository: '@local' + condition: not global.postgres.localCluster + #condition: global.postgres.postgresInit - name: readinessCheck version: ~8.x-0 repository: '@local' diff --git a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml index 0bc7d5bccb..8f904efeae 100644 --- a/kubernetes/cps/components/cps-core/resources/config/application-helm.yml +++ b/kubernetes/cps/components/cps-core/resources/config/application-helm.yml @@ -2,6 +2,7 @@ # Copyright (C) 2021 Pantheon.tech # Modifications Copyright (C) 2020 Bell Canada. # Modifications Copyright (C) 2021 Nordix Foundation. +# Modifications Copyright (C) 2021 Orange # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +19,11 @@ spring: datasource: +{{- if .Values.global.postgres.localCluster }} url: jdbc:postgresql://{{ .Values.postgres.service.name2 }}:5432/{{ .Values.postgres.config.pgDatabase }} +{{- else }} + url: jdbc:postgresql://{{ .Values.global.postgres.service.name2 }}:5432/{{ .Values.postgres.config.pgDatabase }} +{{- end }} username: ${DB_USERNAME} password: ${DB_PASSWORD} driverClassName: org.postgresql.Driver diff --git a/kubernetes/cps/components/cps-core/values.yaml b/kubernetes/cps/components/cps-core/values.yaml index 4f788e7977..55d9fcde66 100644 --- a/kubernetes/cps/components/cps-core/values.yaml +++ b/kubernetes/cps/components/cps-core/values.yaml @@ -52,6 +52,16 @@ global: ingress: virtualhost: baseurl: "simpledemo.onap.org" + #Service Names of the postgres db to connect to. + #Override it to cps-postgres if localCluster is enabled. + postgres: + localCluster: false + service: + name: pgset + name2: tcp-pgset-primary + name3: tcp-pgset-replica + container: + name: postgres image: onap/cps-and-ncmp:2.0.0 containerPort: &svc_port 8080 @@ -206,9 +216,21 @@ postgres: pgUserExternalSecret: *pgUserCredsSecretName pgRootPasswordExternalSecret: *pgRootPassSecretName +postgres-init: + nameOverride: cps-postgres-init + config: + pgUserName: cps + pgDatabase: cpsdb + pgDataPath: data + pgUserExternalSecret: *pgUserCredsSecretName + + # pgPrimaryPassword: password + # pgUserPassword: password + # pgRootPassword: password + readinessCheck: wait_for: - - *postgresName + - '{{ ternary .Values.postgres.service.name "postgres" .Values.global.postgres.localCluster }}' minReadySeconds: 10 updateStrategy: diff --git a/kubernetes/onap/requirements.yaml b/kubernetes/onap/requirements.yaml index 61d4314be4..0a1e769921 100755 --- a/kubernetes/onap/requirements.yaml +++ b/kubernetes/onap/requirements.yaml @@ -1,6 +1,7 @@ # Copyright © 2019 Amdocs, Bell Canada # Copyright (c) 2020 Nordix Foundation, Modifications # Modifications Copyright © 2020 Nokia +# Modifications Copyright © 2021 Orange # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -110,6 +111,10 @@ dependencies: version: ~8.x-0 repository: '@local' condition: portal.enabled + - name: postgres + version: ~8.x-0 + repository: '@local' + condition: postgres.enabled - name: oof version: ~8.x-0 repository: '@local' @@ -169,3 +174,7 @@ dependencies: version: ~8.x-0 repository: '@local' condition: roles-wrapper.enabled + - name: timescaledb + version: ~8.x-0 + repository: '@local' + condition: timescaledb.enabled diff --git a/kubernetes/onap/resources/overrides/onap-all.yaml b/kubernetes/onap/resources/overrides/onap-all.yaml index 91e0157aea..229717a990 100644 --- a/kubernetes/onap/resources/overrides/onap-all.yaml +++ b/kubernetes/onap/resources/overrides/onap-all.yaml @@ -1,6 +1,7 @@ # Copyright © 2019 Amdocs, Bell Canada # Copyright (c) 2020 Nordix Foundation, Modifications # Modifications Copyright © 2020 Nokia +# Modifications Copyright © 2021 Orange # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,6 +25,8 @@ cassandra: enabled: true mariadb-galera: enabled: true +postgres: + enabled: true aaf: enabled: true aai: -- cgit 1.2.3-korg