diff options
30 files changed, 1004 insertions, 20 deletions
diff --git a/kubernetes/clamp/charts/clamp-dash-kibana/values.yaml b/kubernetes/clamp/charts/clamp-dash-kibana/values.yaml index 13b868ab6f..9356c47522 100644 --- a/kubernetes/clamp/charts/clamp-dash-kibana/values.yaml +++ b/kubernetes/clamp/charts/clamp-dash-kibana/values.yaml @@ -34,7 +34,7 @@ busyboxImage: library/busybox:latest # application image repository: nexus3.onap.org:10001 -image: onap/clamp-dashboard-kibana:4.0-STAGING-latest +image: onap/clamp-dashboard-kibana:4.0.1 pullPolicy: Always # flag to enable debugging - application support required diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml b/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml index c81117ffd7..b3d93bb601 100644 --- a/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml +++ b/kubernetes/clamp/charts/clamp-dash-logstash/values.yaml @@ -30,7 +30,7 @@ flavor: small # application image repository: nexus3.onap.org:10001 -image: onap/clamp-dashboard-logstash:4.0-STAGING-latest +image: onap/clamp-dashboard-logstash:4.0.1 pullPolicy: Always # flag to enable debugging - application support required diff --git a/kubernetes/clamp/values.yaml b/kubernetes/clamp/values.yaml index a806e77da8..f4c579b199 100644 --- a/kubernetes/clamp/values.yaml +++ b/kubernetes/clamp/values.yaml @@ -30,7 +30,7 @@ flavor: small # application image repository: nexus3.onap.org:10001 -image: onap/clamp:4.0-STAGING-latest +image: onap/clamp:4.0.1 pullPolicy: Always # flag to enable debugging - application support required diff --git a/kubernetes/common/etcd/.helmignore b/kubernetes/common/etcd/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/kubernetes/common/etcd/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/kubernetes/common/etcd/Chart.yaml b/kubernetes/common/etcd/Chart.yaml new file mode 100644 index 0000000000..31a8ad55f0 --- /dev/null +++ b/kubernetes/common/etcd/Chart.yaml @@ -0,0 +1,23 @@ +# Copyright © 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: etcd +home: https://github.com/coreos/etcd +version: 4.0.0 +appVersion: 2.2.5 +description: Distributed reliable key-value store for the most critical data of a + distributed system. +-icon: https://raw.githubusercontent.com/coreos/etcd/master/logos/etcd-horizontal-color.png +sources: +- https://github.com/coreos/etcd diff --git a/kubernetes/common/etcd/requirements.yaml b/kubernetes/common/etcd/requirements.yaml new file mode 100644 index 0000000000..0ddbcbef4a --- /dev/null +++ b/kubernetes/common/etcd/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright © 2017 Amdocs, Bell Canada +# Modifications Copyright © 2018 Orange +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +dependencies: + - name: common + version: ~4.x-0 + repository: '@local' diff --git a/kubernetes/common/etcd/templates/pv.yaml b/kubernetes/common/etcd/templates/pv.yaml new file mode 100644 index 0000000000..eeaa64598d --- /dev/null +++ b/kubernetes/common/etcd/templates/pv.yaml @@ -0,0 +1,40 @@ +# Copyright © 2018 Amdocs, Bell Canada, AT&T +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{ if .Values.persistence.enabled }} +{{- $root := . -}} +{{ range $i, $e := until (int $root.Values.replicaCount) }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ $root.Release.Name }}-{{ $root.Values.service.name }}-{{ $i }} + namespace: {{ $root.Release.Namespace }} + labels: + type: {{ $root.Values.persistence.storageType }} + app: {{ $root.Values.service.name }} + chart: {{ $root.Chart.Name }}-{{ $root.Chart.Version | replace "+" "_" }} + release: {{ $root.Release.Name }} + heritage: {{ $root.Release.Service }} +spec: + capacity: + storage: {{ $root.Values.persistence.size }} + accessModes: + - {{ $root.Values.persistence.accessMode }} + hostPath: + path: {{ $root.Values.persistence.mountPath }}/{{ $root.Release.Name }}/{{ $root.Values.persistence.mountSubPath }}-{{ $i }} + persistentVolumeReclaimPolicy: {{ $root.Values.persistence.volumeReclaimPolicy }} +{{ end }} +{{ end }} + diff --git a/kubernetes/common/etcd/templates/service.yaml b/kubernetes/common/etcd/templates/service.yaml new file mode 100644 index 0000000000..692faa9f2d --- /dev/null +++ b/kubernetes/common/etcd/templates/service.yaml @@ -0,0 +1,37 @@ +# Copyright 2019 Intel Corporation Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +metadata: + name: {{ include "common.servicename" . }} + labels: + heritage: "{{ .Release.Service }}" + release: "{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ include "common.name" . }} +spec: + ports: + - name: {{ .Values.service.peerPortName }} + port: {{ .Values.service.peerInternalPort }} + - name: {{ .Values.service.clientPortName }} + port: {{ .Values.service.clientInternalPort }} + clusterIP: None + selector: + app: {{ include "common.name" . }} + release: "{{ .Release.Name }}" + diff --git a/kubernetes/common/etcd/templates/statefulset.yaml b/kubernetes/common/etcd/templates/statefulset.yaml new file mode 100644 index 0000000000..ccc6b69971 --- /dev/null +++ b/kubernetes/common/etcd/templates/statefulset.yaml @@ -0,0 +1,244 @@ +# Copyright © 2019 Intel Corporation Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: {{ include "common.servicename" . }} + labels: + heritage: "{{ .Release.Service }}" + release: "{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ template "common.name" . }} +spec: + serviceName: {{ include "common.servicename" . }} + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + heritage: "{{ .Release.Service }}" + release: "{{ .Release.Name }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ include "common.name" . }} + spec: +{{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} +{{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} + containers: + - name: {{ include "common.servicename" . }} + image: "{{ .Values.repository }}/{{ .Values.image }}" + imagePullPolicy: "{{ .Values.pullPolicy }}" + ports: + - containerPort: {{ .Values.service.peerInternalPort }} + name: {{ .Values.service.peerPortName }} + - containerPort: {{ .Values.service.clientInternalPort }} + name: {{ .Values.service.clientPortName }} + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + exec: + command: ["/bin/sh", "-c", "etcdctl cluster-health | grep -w healthy" ] + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + {{ end -}} + readinessProbe: + exec: + command: ["/bin/sh", "-c", "etcdctl cluster-health | grep -w healthy" ] + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + resources: +{{ include "common.resources" . | indent 10 }} + env: + - name: INITIAL_CLUSTER_SIZE + value: {{ .Values.replicaCount | quote }} + - name: SET_NAME + value: {{ include "common.servicename" . }} +{{- if .Values.extraEnv }} +{{ toYaml .Values.extraEnv | indent 8 }} +{{- end }} + lifecycle: + preStop: + exec: + command: + - "/bin/sh" + - "-ec" + - | + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" + done + + HOSTNAME=$(hostname) + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + SET_ID=${HOSTNAME##*[^0-9]} + + if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then + echo "Removing ${HOSTNAME} from etcd cluster" + ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash) + if [ $? -eq 0 ]; then + # Remove everything otherwise the cluster will no longer scale-up + rm -rf /var/run/etcd/* + fi + fi + command: + - "/bin/sh" + - "-ec" + - | + HOSTNAME=$(hostname) + + # store member id into PVC for later member replacement + collect_member() { + while ! etcdctl member list &>/dev/null; do sleep 1; done + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id + exit 0 + } + + eps() { + EPS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379" + done + echo ${EPS} + } + + member_hash() { + etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 + } + + # we should wait for other pods to be up before trying to join + # otherwise we got "no such host" errors when trying to resolve other members + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + while true; do + echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up" + ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break + sleep 1s + done + done + + # re-joining after failure? + if [ -e /var/run/etcd/default.etcd ]; then + echo "Re-joining etcd member" + member_id=$(cat /var/run/etcd/member_id) + + # re-join member + ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 | true + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://0.0.0.0:2380 \ + --listen-client-urls http://0.0.0.0:2379\ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd + fi + + # etcd-SET_ID + SET_ID=${HOSTNAME##*[^0-9]} + + # adding a new member to existing cluster (assuming all initial pods are available) + if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then + export ETCDCTL_ENDPOINT=$(eps) + + # member already added? + MEMBER_HASH=$(member_hash) + if [ -n "${MEMBER_HASH}" ]; then + # the member hash exists but for some reason etcd failed + # as the datadir has not be created, we can remove the member + # and retrieve new hash + etcdctl member remove ${MEMBER_HASH} + fi + + echo "Adding new member" + etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs + + if [ $? -ne 0 ]; then + echo "Exiting" + rm -f /var/run/etcd/new_member_envs + exit 1 + fi + + cat /var/run/etcd/new_member_envs + source /var/run/etcd/new_member_envs + + collect_member & + + exec etcd --name ${HOSTNAME} \ + --listen-peer-urls http://0.0.0.0:2380 \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --data-dir /var/run/etcd/default.etcd \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --initial-cluster ${ETCD_INITIAL_CLUSTER} \ + --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} + fi + + PEERS="" + for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do + PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380" + done + + collect_member & + + # join member + exec etcd --name ${HOSTNAME} \ + --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster ${PEERS} \ + --initial-cluster-state new \ + --data-dir /var/run/etcd/default.etcd + volumeMounts: + - name: {{ include "common.servicename" . }}-datadir + mountPath: /var/run/etcd + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ include "common.servicename" . }}-data + spec: + accessModes: + - "{{ .Values.persistence.accessMode }}" + resources: + requests: + # upstream recommended max is 700M + storage: "{{ .Values.persistence.storage }}" + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" + {{- end }} + {{- end }} + {{- else }} + volumes: + - name: {{ include "common.servicename" . }}-datadir + {{- if .Values.memoryMode }} + emptyDir: + medium: Memory + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} + diff --git a/kubernetes/common/etcd/values.yaml b/kubernetes/common/etcd/values.yaml new file mode 100644 index 0000000000..a999b0c530 --- /dev/null +++ b/kubernetes/common/etcd/values.yaml @@ -0,0 +1,86 @@ +# Copyright © 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + persistence: {} + +################################################################# +# Application configuration defaults. +################################################################# + +#repository: etcd +repository: "k8s.gcr.io" +image: "etcd-amd64:2.2.5" +pullPolicy: Always + +# default number of instances in the StatefulSet +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 90 + periodSeconds: 10 + timeoutSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 90 + periodSeconds: 10 + +persistence: + enabled: false + ## etcd data Persistent Volume Storage Class + ## If defined, storageClassName: <storageClass> + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "-" + accessMode: "ReadWriteOnce" + storage: "1Gi" + mountPath: /dockerdata-nfs + mountSubPath: k8s-etcd + +## This is only available when persistentVolume is false: +## If persistentVolume is not enabled, one can choose to use memory mode for ETCD by setting memoryMode to "true". +## The system will create a volume with "medium: Memory" +memoryMode: false + +service: + name: k8s-etcd + peerInternalPort: 2380 + peerPortName: etcd-server + clientInternalPort : 2379 + clientPortName: etcd-client + +## Node labels and tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature +nodeSelector: {} +tolerations: [] +affinity: {} +extraEnv: [] +resources: {} + diff --git a/kubernetes/dcaegen2/charts/dcae-bootstrap/values.yaml b/kubernetes/dcaegen2/charts/dcae-bootstrap/values.yaml index 6c5bb9a3bd..19e55ace64 100644 --- a/kubernetes/dcaegen2/charts/dcae-bootstrap/values.yaml +++ b/kubernetes/dcaegen2/charts/dcae-bootstrap/values.yaml @@ -97,8 +97,8 @@ default_k8s_location: central # Use to override default setting in blueprints componentImages: dashboard: onap/org.onap.ccsdk.dashboard.ccsdk-app-os:1.1.0-SNAPSHOT-latest - holmes_rules: onap/holmes/rule-management:1.2.4-STAGING-latest - holmes_engine: onap/holmes/engine-management:1.2.3-STAGING-latest + holmes_rules: onap/holmes/rule-management:1.2.6-STAGING-latest + holmes_engine: onap/holmes/engine-management:1.2.5-STAGING-latest tca: onap/org.onap.dcaegen2.deployments.tca-cdap-container:1.1.1 ves: onap/org.onap.dcaegen2.collectors.ves.vescollector:1.4.4 snmptrap: onap/org.onap.dcaegen2.collectors.snmptrap:1.4.0 diff --git a/kubernetes/multicloud/Makefile b/kubernetes/multicloud/Makefile new file mode 100644 index 0000000000..fde29904c7 --- /dev/null +++ b/kubernetes/multicloud/Makefile @@ -0,0 +1,18 @@ +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +make-multicloud: make-multicloud-k8s + +make-multicloud-k8s: + cd charts && helm dep up multicloud-k8s && helm lint multicloud-k8s diff --git a/kubernetes/multicloud/charts/multicloud-k8s/Chart.yaml b/kubernetes/multicloud/charts/multicloud-k8s/Chart.yaml new file mode 100644 index 0000000000..5a73ee32b4 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: ONAP Multicloud Kubernetes Plugin +name: multicloud-k8s +version: 4.0.0 diff --git a/kubernetes/multicloud/charts/multicloud-k8s/requirements.yaml b/kubernetes/multicloud/charts/multicloud-k8s/requirements.yaml new file mode 100644 index 0000000000..566af50cfe --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/requirements.yaml @@ -0,0 +1,25 @@ +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~4.x-0 + # local reference to common chart, as it is + # a part of this chart's package and will not + # be published independently to a repo (at this point) + repository: '@local' + - name: mongo + version: ~4.x-0 + repository: '@local' + diff --git a/kubernetes/multicloud/charts/multicloud-k8s/resources/config/k8sconfig.json b/kubernetes/multicloud/charts/multicloud-k8s/resources/config/k8sconfig.json new file mode 100644 index 0000000000..e451421a5b --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/resources/config/k8sconfig.json @@ -0,0 +1,10 @@ +{ + "ca-file": "/opt/multicloud/k8splugin/certs/root_ca.cer", + "server-cert": "/opt/multicloud/k8splugin/certs/multicloud-k8s.pub", + "server-key": "/opt/multicloud/k8splugin/certs/multicloud-k8s.pr", + "password": "c2VjcmV0bWFuYWdlbWVudHNlcnZpY2VzZWNyZXRwYXNzd29yZA==", + + "database-type": "mongo", + "database-address": "multicloud-k8s-mongo", + "plugin-dir": "/opt/multicloud/k8splugin/plugins" +}
\ No newline at end of file diff --git a/kubernetes/multicloud/charts/multicloud-k8s/templates/configmap.yaml b/kubernetes/multicloud/charts/multicloud-k8s/templates/configmap.yaml new file mode 100644 index 0000000000..c9d55fedf3 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/templates/configmap.yaml @@ -0,0 +1,28 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: +{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} diff --git a/kubernetes/multicloud/charts/multicloud-k8s/templates/deployment.yaml b/kubernetes/multicloud/charts/multicloud-k8s/templates/deployment.yaml new file mode 100644 index 0000000000..846a751336 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/templates/deployment.yaml @@ -0,0 +1,80 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} + spec: + containers: + - image: "{{ include "common.repository" . }}/{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + name: {{ include "common.name" . }} + command: ["/opt/multicloud/k8splugin/k8plugin"] + workingDir: /opt/multicloud/k8splugin + ports: + - containerPort: {{ .Values.service.internalPort }} + {{- if eq .Values.liveness.enabled true }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end -}} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + volumeMounts: + - mountPath: /etc/localtime + name: localtime + readOnly: true + - mountPath: /opt/multicloud/k8splugin/k8sconfig.json + name: {{ include "common.name" .}} + subPath: k8sconfig.json + resources: +{{ include "common.resources" . | indent 10 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 10 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 10 }} + {{- end }} + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name : {{ include "common.name" . }} + configMap: + name: {{ include "common.fullname" . }} + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/multicloud/charts/multicloud-k8s/templates/service.yaml b/kubernetes/multicloud/charts/multicloud-k8s/templates/service.yaml new file mode 100644 index 0000000000..eaf9f4dee5 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/templates/service.yaml @@ -0,0 +1,52 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.servicename" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + msb.onap.org/service-info: '[ + { + "serviceName": "multicloud-k8s", + "version": "v1", + "url": "/", + "protocol": "REST", + "port": "{{ .Values.service.externalPort }}", + "visualRange": "1" + } + ]' +spec: + type: {{ .Values.service.type }} + ports: + - name: {{ .Values.service.PortName }} + {{if eq .Values.service.type "NodePort" -}} + port: {{ .Values.service.internalPort }} + nodePort: {{ .Values.global.nodePortPrefixExt | default "302" }}{{ .Values.service.nodePort }} + {{- else -}} + port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + {{- end}} + protocol: TCP + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} diff --git a/kubernetes/multicloud/charts/multicloud-k8s/values.yaml b/kubernetes/multicloud/charts/multicloud-k8s/values.yaml new file mode 100644 index 0000000000..15aade560d --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-k8s/values.yaml @@ -0,0 +1,108 @@ +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefixExt: 304 + readinessRepository: oomk8s + readinessImage: readiness-check:2.0.0 + loggingRepository: docker.elastic.co + loggingImage: beats/filebeat:5.5.0 + persistence: {} +################################################################# +# Application configuration defaults. +################################################################# +# application image +repository: nexus3.onap.org:10001 +image: onap/multicloud/k8s:0.1.0 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration is via config files + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 10 + periodSeconds: 30 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 10 + periodSeconds: 30 + +service: + type: ClusterIP + name: multicloud-k8s + portName: multicloud-k8s + internalPort: 9015 + externalPort: 9015 + nodePort: 98 + +#Mongo chart overrides for k8splugin +mongo: + nameOverride: multicloud-k8s-mongo + service: + name: multicloud-k8s-mongo + internalPort: 27017 + nfsprovisionerPrefix: multicloud-k8s + sdnctlPrefix: multicloud-k8s + persistence: + mountSubPath: multicloud-k8s/mongo/data + enabled: true + disableNfsProvisioner: true + + +# No persistence right now as we rely on Mongo to handle that +persistence: + enabled: false + volumeReclaimPolicy: Retain + accessMode: ReadWriteOnce + size: 1Gi + mountPath: /dockerdata-nfs + mountSubPath: multicloud-k8s/data + +ingress: + enabled: false + +# Configure resource requests and limits +flavor: large +resources: + small: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 10m + memory: 10Mi + large: + limits: + cpu: 400m + memory: 1Gi + requests: + cpu: 10m + memory: 100Mi + unlimited: {} diff --git a/kubernetes/multicloud/charts/multicloud-starlingx/resources/config/log/config.json b/kubernetes/multicloud/charts/multicloud-starlingx/resources/config/log/config.json new file mode 100644 index 0000000000..d62fec6d85 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-starlingx/resources/config/log/config.json @@ -0,0 +1,70 @@ +{ + "name":"SDCDistributionGroup", + "restServerParameters":{ + "host":"0.0.0.0", + "port":9014, + "userName":"healthcheck", + "password":"zb!XztG34", + "https":true + }, + "receptionHandlerParameters":{ + "SDCReceptionHandler":{ + "receptionHandlerType":"SDC", + "receptionHandlerClassName":"org.onap.policy.distribution.reception.handling.sdc.SdcReceptionHandler", + "receptionHandlerConfigurationName":"sdcConfiguration", + "pluginHandlerParameters":{ + "artifactForwarders":{ + "PAPEngineForwarder":{ + "forwarderType":"PAPEngine", + "forwarderClassName":"org.onap.policy.distribution.forwarding.k8s.K8sArtifactForwarder", + "forwarderConfigurationName": "k8sConfiguration" + } + } + } + } + }, + "receptionHandlerConfigurationParameters":{ + "sdcConfiguration":{ + "parameterClassName":"org.onap.policy.distribution.reception.handling.sdc.SdcReceptionHandlerConfigurationParameterGroup", + "parameters":{ + "asdcAddress": "sdc-be:8443", + "messageBusAddress": [ + "message-router" + ], + "user": "policy", + "password": "Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U", + "pollingInterval":20, + "pollingTimeout":30, + "consumerId": "policy-id", + "artifactTypes": [ + "TOSCA_CSAR", + "HEAT", + "HEAT_ENV", + "CLOUD_TECHNOLOGY_SPECIFIC_ARTIFACT" + ], + "consumerGroup": "policy-group", + "environmentName": "AUTO", + "keystorePath": "null", + "keystorePassword": "null", + "activeserverTlsAuth": false, + "isFilterinEmptyResources": true, + "isUseHttpsWithDmaap": false + } + } + }, + "artifactForwarderConfigurationParameters":{ + "k8sConfiguration":{ + "parameterClassName":"org.onap.policy.distribution.forwarding.k8s.K8sArtifactForwarderParameterGroup", + "parameters":{ + "useHttps": true, + "hostname": "pdp", + "port": 8081, + "userName": "testpdp", + "password": "alpha123", + "clientAuth": "cHl0aG9uOnRlc3Q=", + "isManaged": true, + "pdpGroup": "default" + } + } + } +} diff --git a/kubernetes/multicloud/charts/multicloud-starlingx/templates/deployment.yaml b/kubernetes/multicloud/charts/multicloud-starlingx/templates/deployment.yaml index 0ccc3f9f95..62bacd7da5 100644 --- a/kubernetes/multicloud/charts/multicloud-starlingx/templates/deployment.yaml +++ b/kubernetes/multicloud/charts/multicloud-starlingx/templates/deployment.yaml @@ -61,6 +61,8 @@ spec: - mountPath: /opt/starlingx/starlingx/pub/config/log.yml name: starlingx-logconfig subPath: log.yml + - mountPath: /opt/artifacts/ + name: artifact-data resources: {{ include "common.resources" . | indent 12 }} image: "{{ include "common.repository" . }}/{{ .Values.image }}" @@ -93,6 +95,19 @@ spec: name: starlingx-log - mountPath: /usr/share/filebeat/data name: starlingx-data-filebeat + - image: "{{ include "common.repository" . }}/{{ .Values.global.artifactImage }}" + name: framework-artifactbroker + command: ["/opt/app/distribution/bin/artifact-dist.sh"] + args: ["/opt/app/distribution/etc/mounted/config.json"] + ports: + - containerPort: 9014 + protocol: TCP + volumeMounts: + - mountPath: /opt/app/distribution/etc/mounted/config.json + name: starlingx-logconfig + subPath: config.json + - mountPath: /data + name: artifact-data volumes: - name: starlingx-log @@ -105,6 +120,8 @@ spec: - name: starlingx-logconfig configMap: name: {{ include "common.fullname" . }}-log-configmap + - name: artifact-data + emptyDir: {} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" restartPolicy: Always diff --git a/kubernetes/multicloud/charts/multicloud-windriver/resources/config/log/config.json b/kubernetes/multicloud/charts/multicloud-windriver/resources/config/log/config.json new file mode 100644 index 0000000000..d62fec6d85 --- /dev/null +++ b/kubernetes/multicloud/charts/multicloud-windriver/resources/config/log/config.json @@ -0,0 +1,70 @@ +{ + "name":"SDCDistributionGroup", + "restServerParameters":{ + "host":"0.0.0.0", + "port":9014, + "userName":"healthcheck", + "password":"zb!XztG34", + "https":true + }, + "receptionHandlerParameters":{ + "SDCReceptionHandler":{ + "receptionHandlerType":"SDC", + "receptionHandlerClassName":"org.onap.policy.distribution.reception.handling.sdc.SdcReceptionHandler", + "receptionHandlerConfigurationName":"sdcConfiguration", + "pluginHandlerParameters":{ + "artifactForwarders":{ + "PAPEngineForwarder":{ + "forwarderType":"PAPEngine", + "forwarderClassName":"org.onap.policy.distribution.forwarding.k8s.K8sArtifactForwarder", + "forwarderConfigurationName": "k8sConfiguration" + } + } + } + } + }, + "receptionHandlerConfigurationParameters":{ + "sdcConfiguration":{ + "parameterClassName":"org.onap.policy.distribution.reception.handling.sdc.SdcReceptionHandlerConfigurationParameterGroup", + "parameters":{ + "asdcAddress": "sdc-be:8443", + "messageBusAddress": [ + "message-router" + ], + "user": "policy", + "password": "Kp8bJ4SXszM0WXlhak3eHlcse2gAw84vaoGGmJvUy2U", + "pollingInterval":20, + "pollingTimeout":30, + "consumerId": "policy-id", + "artifactTypes": [ + "TOSCA_CSAR", + "HEAT", + "HEAT_ENV", + "CLOUD_TECHNOLOGY_SPECIFIC_ARTIFACT" + ], + "consumerGroup": "policy-group", + "environmentName": "AUTO", + "keystorePath": "null", + "keystorePassword": "null", + "activeserverTlsAuth": false, + "isFilterinEmptyResources": true, + "isUseHttpsWithDmaap": false + } + } + }, + "artifactForwarderConfigurationParameters":{ + "k8sConfiguration":{ + "parameterClassName":"org.onap.policy.distribution.forwarding.k8s.K8sArtifactForwarderParameterGroup", + "parameters":{ + "useHttps": true, + "hostname": "pdp", + "port": 8081, + "userName": "testpdp", + "password": "alpha123", + "clientAuth": "cHl0aG9uOnRlc3Q=", + "isManaged": true, + "pdpGroup": "default" + } + } + } +} diff --git a/kubernetes/multicloud/charts/multicloud-windriver/templates/deployment.yaml b/kubernetes/multicloud/charts/multicloud-windriver/templates/deployment.yaml index 99b1ea9557..e6f1456874 100644 --- a/kubernetes/multicloud/charts/multicloud-windriver/templates/deployment.yaml +++ b/kubernetes/multicloud/charts/multicloud-windriver/templates/deployment.yaml @@ -62,6 +62,8 @@ spec: - mountPath: /opt/windriver/titanium_cloud/pub/config/log.yml name: windriver-logconfig subPath: log.yml + - mountPath: /opt/artifacts/ + name: artifact-data resources: {{ include "common.resources" . | indent 12 }} image: "{{ include "common.repository" . }}/{{ .Values.image }}" @@ -94,6 +96,19 @@ spec: name: windriver-log - mountPath: /usr/share/filebeat/data name: windriver-data-filebeat + - image: "{{ include "common.repository" . }}/{{ .Values.global.artifactImage }}" + name: framework-artifactbroker + command: ["/opt/app/distribution/bin/artifact-dist.sh"] + args: ["/opt/app/distribution/etc/mounted/config.json"] + ports: + - containerPort: 9014 + protocol: TCP + volumeMounts: + - mountPath: /opt/app/distribution/etc/mounted/config.json + name: windriver-logconfig + subPath: config.json + - mountPath: /data + name: artifact-data volumes: - name: windriver-log @@ -106,6 +121,8 @@ spec: - name: windriver-logconfig configMap: name: {{ include "common.fullname" . }}-log-configmap + - name: artifact-data + emptyDir: {} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" restartPolicy: Always diff --git a/kubernetes/multicloud/values.yaml b/kubernetes/multicloud/values.yaml index 6188e43fb7..889c06bd47 100644 --- a/kubernetes/multicloud/values.yaml +++ b/kubernetes/multicloud/values.yaml @@ -20,6 +20,7 @@ global: nodePortPrefix: 302 loggingRepository: docker.elastic.co loggingImage: beats/filebeat:5.5.0 + artifactImage: onap/multicloud/framework-artifactbroker:1.3.1 prometheus: enabled: false diff --git a/kubernetes/readiness/docker/init/Dockerfile b/kubernetes/readiness/docker/init/Dockerfile index 7214e04174..b060915354 100644 --- a/kubernetes/readiness/docker/init/Dockerfile +++ b/kubernetes/readiness/docker/init/Dockerfile @@ -1,17 +1,12 @@ -from ubuntu:16.04 +FROM python:2-alpine ENV no_proxy "localhost,127.0.0.1,.cluster.local,$KUBERNETES_SERVICE_HOST" # Setup Corporate proxy ENV https_proxy ${HTTPS_PROXY} ENV http_proxy ${HTTP_PROXY} -# Additional packages -RUN apt-get update -RUN apt-get install -y apt-utils git wget curl dnsutils python python-pip jq net-tools coreutils vim - RUN pip install requests pyyaml kubernetes - ENV CERT="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" ENV TOKEN="/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -21,4 +16,5 @@ RUN chmod a+x /root/ready.py COPY job_complete.py /root/job_complete.py RUN chmod a+x /root/job_complete.py -ENTRYPOINT /root/ready.py +ENTRYPOINT ["/root/ready.py"] +CMD [""]
\ No newline at end of file diff --git a/kubernetes/readiness/docker/init/job_complete.py b/kubernetes/readiness/docker/init/job_complete.py index 97d22da6c1..b20cb5adec 100644 --- a/kubernetes/readiness/docker/init/job_complete.py +++ b/kubernetes/readiness/docker/init/job_complete.py @@ -1,9 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/env python import getopt import logging import os import sys import time +import random from kubernetes import client @@ -91,7 +92,8 @@ def main(argv): log.warning("timed out waiting for '" + job_name + "' to be completed") exit(1) else: - time.sleep(5) + # spread in time potentially parallel execution in multiple containers + time.sleep(random.randint(5, 11)) if __name__ == "__main__": main(sys.argv[1:])
\ No newline at end of file diff --git a/kubernetes/readiness/docker/init/ready.py b/kubernetes/readiness/docker/init/ready.py index 87c09a444c..0e133eab01 100644 --- a/kubernetes/readiness/docker/init/ready.py +++ b/kubernetes/readiness/docker/init/ready.py @@ -1,9 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/env python import getopt import logging import os import sys import time +import random from kubernetes import client @@ -154,7 +155,8 @@ def main(argv): log.warning("timed out waiting for '" + container_name + "' to be ready") exit(1) else: - time.sleep(5) + # spread in time potentially parallel execution in multiple containers + time.sleep(random.randint(5, 11)) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/kubernetes/sdc/charts/sdc-wfd-fe/templates/deployment.yaml b/kubernetes/sdc/charts/sdc-wfd-fe/templates/deployment.yaml index c1babf3063..0be06f3985 100644 --- a/kubernetes/sdc/charts/sdc-wfd-fe/templates/deployment.yaml +++ b/kubernetes/sdc/charts/sdc-wfd-fe/templates/deployment.yaml @@ -51,6 +51,7 @@ spec: imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} ports: - containerPort: {{ .Values.service.internalPort }} + - containerPort: {{ .Values.service.internalPort2 }} {{ if .Values.liveness.enabled }} livenessProbe: tcpSocket: diff --git a/kubernetes/vfc/charts/vfc-generic-vnfm-driver/values.yaml b/kubernetes/vfc/charts/vfc-generic-vnfm-driver/values.yaml index 47145fc9ec..2134309f46 100644 --- a/kubernetes/vfc/charts/vfc-generic-vnfm-driver/values.yaml +++ b/kubernetes/vfc/charts/vfc-generic-vnfm-driver/values.yaml @@ -29,7 +29,7 @@ global: flavor: small repository: nexus3.onap.org:10001 -image: onap/vfc/gvnfmdriver:1.3.0 +image: onap/vfc/gvnfmdriver:1.3.1 pullPolicy: Always #Istio sidecar injection policy @@ -87,4 +87,4 @@ resources: requests: cpu: 200m memory: 500Mi - unlimited: {}
\ No newline at end of file + unlimited: {} diff --git a/kubernetes/vfc/charts/vfc-juju-vnfm-driver/values.yaml b/kubernetes/vfc/charts/vfc-juju-vnfm-driver/values.yaml index 8833a5a627..f68f0fc790 100644 --- a/kubernetes/vfc/charts/vfc-juju-vnfm-driver/values.yaml +++ b/kubernetes/vfc/charts/vfc-juju-vnfm-driver/values.yaml @@ -29,7 +29,7 @@ global: flavor: small repository: nexus3.onap.org:10001 -image: onap/vfc/jujudriver:1.3.0 +image: onap/vfc/jujudriver:1.3.1 pullPolicy: Always #Istio sidecar injection policy @@ -88,4 +88,4 @@ resources: requests: cpu: 200m memory: 2000Mi - unlimited: {}
\ No newline at end of file + unlimited: {} |