From 480a77d0c83b6ae227b3d992763eb88bb2520d55 Mon Sep 17 00:00:00 2001 From: efiacor Date: Mon, 19 Jul 2021 12:25:23 +0100 Subject: [DMAAP-MR] Migrate csit suite to the repo Signed-off-by: efiacor Change-Id: I4b623aec5f9a90acbbc1895a144ad649f793b6a3 Issue-ID: DMAAP-1637 --- .gitignore | 4 +- csit/plans/mrpubsub/setup.sh | 27 +++ csit/plans/mrpubsub/teardown.sh | 22 +++ csit/plans/mrpubsub/testplan.txt | 2 + csit/prepare-csit.sh | 49 +++++ csit/run-csit.sh | 196 +++++++++++++++++++ csit/run-project-csit.sh | 28 +++ .../dmaap-message-router/dmaap-mr-launch.sh | 66 +++++++ .../dmaap-message-router/dmaap-mr-teardown.sh | 29 +++ .../docker-compose/docker-compose.yml | 76 ++++++++ .../docker-compose/kafka/zk_client_jaas.conf | 5 + .../docker-compose/mr/MsgRtrApi.properties | 166 ++++++++++++++++ .../docker-compose/mr/cadi.properties | 19 ++ .../docker-compose/mr/logback.xml | 208 +++++++++++++++++++++ .../docker-compose/zk/zk_server_jaas.conf | 4 + csit/scripts/get-instance-ip.sh | 17 ++ csit/tests/mrpubsub/mrpubsub.robot | 74 ++++++++ pom.xml | 6 +- .../ajsc/dmaap_v1/dmaap/v1/conf/jaxrsBeans.groovy | 2 +- .../docker-compose/docker-compose-old.yml | 57 ------ .../resources/docker-compose/docker-compose.yml | 73 -------- .../docker-compose/kafka/zk_client_jaas.conf | 5 - .../docker-compose/mr/MsgRtrApi.properties | 172 ----------------- .../resources/docker-compose/mr/cadi.properties | 19 -- src/main/resources/docker-compose/mr/logback.xml | 208 --------------------- .../docker-compose/zk/zk_server_jaas.conf | 4 - 26 files changed, 995 insertions(+), 543 deletions(-) create mode 100755 csit/plans/mrpubsub/setup.sh create mode 100755 csit/plans/mrpubsub/teardown.sh create mode 100644 csit/plans/mrpubsub/testplan.txt create mode 100755 csit/prepare-csit.sh create mode 100755 csit/run-csit.sh create mode 100755 csit/run-project-csit.sh create mode 100755 csit/scripts/dmaap-message-router/dmaap-mr-launch.sh create mode 100755 csit/scripts/dmaap-message-router/dmaap-mr-teardown.sh create mode 100644 csit/scripts/dmaap-message-router/docker-compose/docker-compose.yml create mode 100644 csit/scripts/dmaap-message-router/docker-compose/kafka/zk_client_jaas.conf create mode 100644 csit/scripts/dmaap-message-router/docker-compose/mr/MsgRtrApi.properties create mode 100644 csit/scripts/dmaap-message-router/docker-compose/mr/cadi.properties create mode 100644 csit/scripts/dmaap-message-router/docker-compose/mr/logback.xml create mode 100644 csit/scripts/dmaap-message-router/docker-compose/zk/zk_server_jaas.conf create mode 100755 csit/scripts/get-instance-ip.sh create mode 100755 csit/tests/mrpubsub/mrpubsub.robot delete mode 100644 src/main/resources/docker-compose/docker-compose-old.yml delete mode 100644 src/main/resources/docker-compose/docker-compose.yml delete mode 100644 src/main/resources/docker-compose/kafka/zk_client_jaas.conf delete mode 100644 src/main/resources/docker-compose/mr/MsgRtrApi.properties delete mode 100644 src/main/resources/docker-compose/mr/cadi.properties delete mode 100644 src/main/resources/docker-compose/mr/logback.xml delete mode 100644 src/main/resources/docker-compose/zk/zk_server_jaas.conf diff --git a/.gitignore b/.gitignore index 97e81c5..a679fe5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ *.iml /target/* .idea/* -/src/.idea/* \ No newline at end of file +/src/.idea/* +/venv/* +*/archives/ \ No newline at end of file diff --git a/csit/plans/mrpubsub/setup.sh b/csit/plans/mrpubsub/setup.sh new file mode 100755 index 0000000..411343a --- /dev/null +++ b/csit/plans/mrpubsub/setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# ============LICENSE_START======================================================= +# ONAP DMAAP MR +# ================================================================================ +# Copyright (C) 2017 AT&T Intellectual Property. All rights reserved. +# Modification copyright (C) 2021 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# =================================================================== +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +source ${WORKSPACE}/scripts/dmaap-message-router/dmaap-mr-launch.sh +dmaap_mr_launch +#Pass any variables required by Robot test suites in ROBOT_VARIABLES +ROBOT_VARIABLES="-v DMAAP_MR_IP:${DMAAP_MR_IP}" diff --git a/csit/plans/mrpubsub/teardown.sh b/csit/plans/mrpubsub/teardown.sh new file mode 100755 index 0000000..65d4884 --- /dev/null +++ b/csit/plans/mrpubsub/teardown.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# Modification copyright (C) 2021 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2017 AT&T Intellectual Property +# + +source ${WORKSPACE}/scripts/dmaap-message-router/dmaap-mr-teardown.sh +dmaap_mr_teardown diff --git a/csit/plans/mrpubsub/testplan.txt b/csit/plans/mrpubsub/testplan.txt new file mode 100644 index 0000000..e1d4098 --- /dev/null +++ b/csit/plans/mrpubsub/testplan.txt @@ -0,0 +1,2 @@ +# Place the suites in run order. +mrpubsub diff --git a/csit/prepare-csit.sh b/csit/prepare-csit.sh new file mode 100755 index 0000000..a65368c --- /dev/null +++ b/csit/prepare-csit.sh @@ -0,0 +1,49 @@ +#!/bin/bash -x +# +# Copyright 2019 © Samsung Electronics Co., Ltd. +# Modification copyright (C) 2021 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script installs common libraries required by CSIT tests +# + +if [ -z "$WORKSPACE" ]; then + export WORKSPACE=`git rev-parse --show-toplevel` +fi + +TESTPLANDIR=${WORKSPACE}/${TESTPLAN} + +# Assume that if ROBOT_VENV is set and virtualenv with system site packages can be activated, +# ci-management/jjb/integration/include-raw-integration-install-robotframework.sh has already +# been executed + +if [ -f ${WORKSPACE}/env.properties ]; then + source ${WORKSPACE}/env.properties +fi +if [ -f ${ROBOT_VENV}/bin/activate ]; then + source ${ROBOT_VENV}/bin/activate +else + rm -rf /tmp/ci-management + rm -f ${WORKSPACE}/env.properties + cd /tmp + git clone "https://gerrit.onap.org/r/ci-management" + source /tmp/ci-management/jjb/integration/include-raw-integration-install-robotframework.sh +fi + +# install eteutils +mkdir -p ${ROBOT_VENV}/src/onap +rm -rf ${ROBOT_VENV}/src/onap/testsuite +pip install --upgrade --extra-index-url="https://nexus3.onap.org/repository/PyPi.staging/simple" 'robotframework-onap==0.5.1.*' --pre + +pip freeze diff --git a/csit/run-csit.sh b/csit/run-csit.sh new file mode 100755 index 0000000..b987e60 --- /dev/null +++ b/csit/run-csit.sh @@ -0,0 +1,196 @@ +#!/bin/bash -x +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# Modification Copyright 2019 © Samsung Electronics Co., Ltd. +# Modification copyright (C) 2021 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# $1 project/functionality +# $2 robot options + +# +# functions +# + +function on_exit(){ + rc=$? + if [[ ${WORKSPACE} ]]; then + if [[ ${WORKDIR} ]]; then + rsync -av "$WORKDIR/" "$WORKSPACE/archives/$TESTPLAN" + fi + # Record list of active docker containers + docker ps --format "{{.Image}}" > "$WORKSPACE/archives/$TESTPLAN/_docker-images.log" + + # show memory consumption after all docker instances initialized + docker_stats | tee "$WORKSPACE/archives/$TESTPLAN/_sysinfo-2-after-robot.txt" + fi + # Run teardown script plan if it exists + cd "${TESTPLANDIR}" + TEARDOWN="${TESTPLANDIR}/teardown.sh" + if [ -f "${TEARDOWN}" ]; then + echo "Running teardown script ${TEARDOWN}" + source_safely "${TEARDOWN}" + fi + # TODO: do something with the output + exit $rc +} +# ensure that teardown and other finalizing steps are always executed +trap on_exit EXIT + +function docker_stats(){ + #General memory details + echo "> top -bn1 | head -3" + top -bn1 | head -3 + echo + + echo "> free -h" + free -h + echo + + #Memory details per Docker + echo "> docker ps" + docker ps + echo + + echo "> docker stats --no-stream" + docker stats --no-stream + echo +} + +# save current set options +function save_set() { + RUN_CSIT_SAVE_SET="$-" + RUN_CSIT_SHELLOPTS="$SHELLOPTS" +} + +# load the saved set options +function load_set() { + _setopts="$-" + + # bash shellopts + for i in $(echo "$SHELLOPTS" | tr ':' ' ') ; do + set +o ${i} + done + for i in $(echo "$RUN_CSIT_SHELLOPTS" | tr ':' ' ') ; do + set -o ${i} + done + + # other options + for i in $(echo "$_setopts" | sed 's/./& /g') ; do + set +${i} + done + set -${RUN_CSIT_SAVE_SET} +} + +# set options for quick bailout when error +function harden_set() { + set -xeo pipefail + set +u # enabled it would probably fail too many often +} + +# relax set options so the sourced file will not fail +# the responsibility is shifted to the sourced file... +function relax_set() { + set +e + set +o pipefail +} + +# wrapper for sourcing a file +function source_safely() { + [ -z "$1" ] && return 1 + relax_set + . "$1" + load_set +} + +# +# main +# + +# set and save options for quick failure +harden_set && save_set + +if [ $# -eq 0 ] +then + echo + echo "Usage: $0 plans// []" + echo + echo " , , : " + echo " The same values as for the '{project}-csit-{functionality}' JJB job template." + echo + exit 1 +fi + +if [ -z "$WORKSPACE" ]; then + export WORKSPACE=$(git rev-parse --show-toplevel) +fi + +if [ -f "${WORKSPACE}/${1}/testplan.txt" ]; then + export TESTPLAN="${1}" +else + echo "testplan not found: ${WORKSPACE}/${TESTPLAN}/testplan.txt" + exit 2 +fi + +export TESTOPTIONS="${2}" + +rm -rf "$WORKSPACE/archives/$TESTPLAN" +mkdir -p "$WORKSPACE/archives/$TESTPLAN" + +TESTPLANDIR="${WORKSPACE}/${TESTPLAN}" + +# Run installation of prerequired libraries +source_safely "${WORKSPACE}/prepare-csit.sh" + +# Activate the virtualenv containing all the required libraries installed by prepare-csit.sh +source_safely "${ROBOT_VENV}/bin/activate" + +WORKDIR=$(mktemp -d --suffix=-robot-workdir) +cd "${WORKDIR}" + +# Add csit scripts to PATH +export PATH="${PATH}:${WORKSPACE}/docker/scripts:${WORKSPACE}/scripts:${ROBOT_VENV}/bin" +export SCRIPTS="${WORKSPACE}/scripts" +export ROBOT_VARIABLES= + +# Sign in to nexus3 docker repo +docker login -u docker -p docker nexus3.onap.org:10001 + +# Run setup script plan if it exists +cd "${TESTPLANDIR}" +SETUP="${TESTPLANDIR}/setup.sh" +if [ -f "${SETUP}" ]; then + echo "Running setup script ${SETUP}" + source_safely "${SETUP}" +fi + +# show memory consumption after all docker instances initialized +docker_stats | tee "$WORKSPACE/archives/$TESTPLAN/_sysinfo-1-after-setup.txt" + +# Run test plan +cd "$WORKDIR" +echo "Reading the testplan:" +cat "${TESTPLANDIR}/testplan.txt" | egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' | sed "s|^|${WORKSPACE}/tests/|" > testplan.txt +cat testplan.txt +SUITES=$( xargs -a testplan.txt ) + +echo ROBOT_VARIABLES="${ROBOT_VARIABLES}" +echo "Starting Robot test suites ${SUITES} ..." +relax_set +python -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES} +RESULT=$? +load_set +echo "RESULT: $RESULT" +# Note that the final steps are done in on_exit function after this exit! +exit $RESULT diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh new file mode 100755 index 0000000..7232db1 --- /dev/null +++ b/csit/run-project-csit.sh @@ -0,0 +1,28 @@ +#!/bin/bash -x +# +# Copyright 2020-2021 © Samsung Electronics Co., Ltd. +# Modifications copyright (C) 2021 Nordix Foundation.. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export WORKSPACE=$(git rev-parse --show-toplevel)/csit + +rm -rf ${WORKSPACE}/archives +mkdir -p ${WORKSPACE}/archives +cd ${WORKSPACE} + +./run-csit.sh plans/mrpubsub + + + diff --git a/csit/scripts/dmaap-message-router/dmaap-mr-launch.sh b/csit/scripts/dmaap-message-router/dmaap-mr-launch.sh new file mode 100755 index 0000000..de462a1 --- /dev/null +++ b/csit/scripts/dmaap-message-router/dmaap-mr-launch.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# ============LICENSE_START======================================================= +# ONAP DMAAP MR +# ================================================================================ +# Copyright (C) 2018 AT&T Intellectual Property. All rights reserved. +# Modification copyright (C) 2021 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END============================================ +# =================================================================== +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +# function to launch DMaaP MR docker containers. +# sets global var DMAAP_MR_IP with assigned IP address of MR container. +# (kafka and zk containers are not called externally) + +function dmaap_mr_launch() { + mkdir -p $WORKSPACE/archives/dmaap/mr/last_run_logs + + # start DMaaP MR containers with docker compose and configuration from docker-compose.yml + docker login -u docker -p docker nexus3.onap.org:10001 + docker-compose -f ${WORKSPACE}/scripts/dmaap-message-router/docker-compose/docker-compose.yml up -d + docker ps + + # Wait for initialization of Docker containers for DMaaP MR, Kafka and Zookeeper + for i in {1..50}; do + if [[ $(docker inspect --format '{{ .State.Running }}' dmaap-mr) ]] && \ + [[ $(docker inspect --format '{{ .State.Running }}' dmaap-kafka) ]] && \ + [[ $(docker inspect --format '{{ .State.Running }}' dmaap-zookeeper) ]] + then + echo "DMaaP Service Running" + break + else + echo sleep $i + sleep $i + fi + done + +DMAAP_MR_IP=`get-instance-ip.sh dmaap-mr` +echo MR_IP=${DMAAP_MR_IP} +export DMAAP_MR_IP + +echo "Waiting for dmaap-message-router to come up ..." +for i in {1..20}; do + dmaap_state=$(curl --write-out '%{http_code}' --silent --output /dev/null $DMAAP_MR_IP:3904/topics) + if [ ${dmaap_state} == "200" ] + then + break + else + sleep 5 + fi +done + +} + diff --git a/csit/scripts/dmaap-message-router/dmaap-mr-teardown.sh b/csit/scripts/dmaap-message-router/dmaap-mr-teardown.sh new file mode 100755 index 0000000..1389142 --- /dev/null +++ b/csit/scripts/dmaap-message-router/dmaap-mr-teardown.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# Modification copyright (C) 2021 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Modifications copyright (c) 2018 AT&T Intellectual Property +# + +function dmaap_mr_teardown() { + cd ${WORKSPACE}/archives/dmaap/mr + rm -rf last_run_logs/* + docker cp dmaap-mr:/appl/logs/EELF last_run_logs/dmaap_mr_logs + docker cp dmaap-kafka:/var/lib/kafka/data/ last_run_logs/kafka_logs + docker logs dmaap-kafka > last_run_logs/kafka_logs/kafka.log + docker logs dmaap-zookeeper > last_run_logs/zookeeper.log + docker-compose -f ${WORKSPACE}/scripts/dmaap-message-router/docker-compose/docker-compose.yml rm -sf +} diff --git a/csit/scripts/dmaap-message-router/docker-compose/docker-compose.yml b/csit/scripts/dmaap-message-router/docker-compose/docker-compose.yml new file mode 100644 index 0000000..d225499 --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/docker-compose.yml @@ -0,0 +1,76 @@ +version: '2.1' +services: + zookeeper: + image: nexus3.onap.org:10001/onap/dmaap/zookeeper:6.1.0 + container_name: dmaap-zookeeper + ports: + - "2181:2181" + environment: + ZOOKEEPER_REPLICAS: 1 + ZOOKEEPER_TICK_TIME: 2000 + ZOOKEEPER_SYNC_LIMIT: 5 + ZOOKEEPER_INIT_LIMIT: 10 + ZOOKEEPER_MAX_CLIENT_CNXNS: 200 + ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT: 3 + ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: 24 + ZOOKEEPER_CLIENT_PORT: 2181 + KAFKA_OPTS: -Djava.security.auth.login.config=/etc/zookeeper/secrets/jaas/zk_server_jaas.conf -Dzookeeper.kerberos.removeHostFromPrincipal=true -Dzookeeper.kerberos.removeRealmFromPrincipal=true -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dzookeeper.requireClientAuthScheme=sasl -Dzookeeper.4lw.commands.whitelist=* + ZOOKEEPER_SERVER_ID: 1 + volumes: + - ./zk/zk_server_jaas.conf:/etc/zookeeper/secrets/jaas/zk_server_jaas.conf + networks: + net: + aliases: + - zookeeper + + kafka: + image: nexus3.onap.org:10001/onap/dmaap/kafka111:1.1.0 + container_name: dmaap-kafka + ports: + - "9092:9092" + environment: + enableCadi: 'false' + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 40000 + KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 40000 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL_PLAINTEXT:PLAINTEXT,EXTERNAL_PLAINTEXT:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: INTERNAL_PLAINTEXT://kafka:9092 + KAFKA_LISTENERS: INTERNAL_PLAINTEXT://0.0.0.0:9092 + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL_PLAINTEXT + KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' + KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/jaas/zk_client_jaas.conf + KAFKA_ZOOKEEPER_SET_ACL: 'true' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + # Reduced the number of partitions only to avoid the timeout error for the first subscribe call in slow environment + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 + volumes: + - ./kafka/zk_client_jaas.conf:/etc/kafka/secrets/jaas/zk_client_jaas.conf + networks: + net: + aliases: + - kafka + depends_on: + - zookeeper + + dmaap: + image: onap/dmaap/dmaap-mr:latest + container_name: dmaap-mr + ports: + - "3904:3904" + - "3905:3905" + environment: + enableCadi: 'false' + volumes: + - ./mr/MsgRtrApi.properties:/appl/dmaapMR1/bundleconfig/etc/appprops/MsgRtrApi.properties + - ./mr/logback.xml:/appl/dmaapMR1/bundleconfig/etc/logback.xml + - ./mr/cadi.properties:/appl/dmaapMR1/etc/cadi.properties + networks: + net: + aliases: + - dmaap + depends_on: + - zookeeper + - kafka +networks: + net: + driver: bridge diff --git a/csit/scripts/dmaap-message-router/docker-compose/kafka/zk_client_jaas.conf b/csit/scripts/dmaap-message-router/docker-compose/kafka/zk_client_jaas.conf new file mode 100644 index 0000000..79a7601 --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/kafka/zk_client_jaas.conf @@ -0,0 +1,5 @@ +Client { + org.apache.zookeeper.server.auth.DigestLoginModule required + username="kafka" + password="kafka_secret"; + }; \ No newline at end of file diff --git a/csit/scripts/dmaap-message-router/docker-compose/mr/MsgRtrApi.properties b/csit/scripts/dmaap-message-router/docker-compose/mr/MsgRtrApi.properties new file mode 100644 index 0000000..4764321 --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/mr/MsgRtrApi.properties @@ -0,0 +1,166 @@ +# LICENSE_START======================================================= +# org.onap.dmaap +# ================================================================================ +# Copyright © 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. +# +############################################################################### +############################################################################### +## +## Cambria API Server config +## +## Default values are shown as commented settings. +## +############################################################################### +## +## HTTP service +## +## 3904 is standard as of 7/29/14. +# +## Zookeeper Connection +## +## Both Cambria and Kafka make use of Zookeeper. +## +config.zk.servers=zookeeper:2181 + +############################################################################### +## +## Kafka Connection +## +## Items below are passed through to Kafka's producer and consumer +## configurations (after removing "kafka.") +## if you want to change request.required.acks it can take this one value +#kafka.metadata.broker.list=localhost:9092,localhost:9093 +#kafka.metadata.broker.list={{.Values.kafka.name}}:{{.Values.kafka.port}} +kafka.metadata.broker.list=kafka:9092 +##kafka.request.required.acks=-1 +#kafka.client.zookeeper=${config.zk.servers} +consumer.timeout.ms=100 +zookeeper.connection.timeout.ms=6000 +zookeeper.session.timeout.ms=20000 +zookeeper.sync.time.ms=2000 +auto.commit.interval.ms=1000 +fetch.message.max.bytes =1000000 +auto.commit.enable=false + +#(backoff*retries > zksessiontimeout) +kafka.rebalance.backoff.ms=10000 +kafka.rebalance.max.retries=6 + + +############################################################################### +## +## Secured Config +## +## Some data stored in the config system is sensitive -- API keys and secrets, +## for example. to protect it, we use an encryption layer for this section +## of the config. +## +## The key is a base64 encode AES key. This must be created/configured for +## each installation. +#cambria.secureConfig.key= +## +## The initialization vector is a 16 byte value specific to the secured store. +## This must be created/configured for each installation. +#cambria.secureConfig.iv= + +## Southfield Sandbox +cambria.secureConfig.key=b/7ouTn9FfEw2PQwL0ov/Q== +cambria.secureConfig.iv=wR9xP5k5vbz/xD0LmtqQLw== +authentication.adminSecret=fe3cCompound + + +############################################################################### +## +## Consumer Caching +## +## Kafka expects live connections from the consumer to the broker, which +## obviously doesn't work over connectionless HTTP requests. The Cambria +## server proxies HTTP requests into Kafka consumer sessions that are kept +## around for later re-use. Not doing so is costly for setup per request, +## which would substantially impact a high volume consumer's performance. +## +## This complicates Cambria server failover, because we often need server +## A to close its connection before server B brings up the replacement. +## + +## The consumer cache is normally enabled. +#cambria.consumer.cache.enabled=true + +## Cached consumers are cleaned up after a period of disuse. The server inspects +## consumers every sweepFreqSeconds and will clean up any connections that are +## dormant for touchFreqMs. +#cambria.consumer.cache.sweepFreqSeconds=15 +cambria.consumer.cache.touchFreqMs=120000 +##stickforallconsumerrequests=false +## The cache is managed through ZK. The default value for the ZK connection +## string is the same as config.zk.servers. +#cambria.consumer.cache.zkConnect=${config.zk.servers} + +## +## Shared cache information is associated with this node's name. The default +## name is the hostname plus the HTTP service port this host runs on. (The +## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(), +## which is not always adequate.) You can set this value explicitly here. +## +#cambria.api.node.identifier= + +#cambria.rateLimit.maxEmptyPollsPerMinute=30 +#cambria.rateLimitActual.delay.ms=10 + +############################################################################### +## +## Metrics Reporting +## +## This server can report its metrics periodically on a topic. +## +#metrics.send.cambria.enabled=true +#metrics.send.cambria.topic=cambria.apinode.metrics +#msgrtr.apinode.metrics.dmaap +#metrics.send.cambria.sendEverySeconds=60 + +cambria.consumer.cache.zkBasePath=/fe3c/cambria/consumerCache +consumer.timeout=17 +default.partitions=3 +default.replicas=3 +############################################################################## +#100mb +maxcontentlength=10000 + + +############################################################################## +#AAF Properties +msgRtr.namespace.aaf=org.onap.dmaap.mr.topic +msgRtr.topicfactory.aaf=org.onap.dmaap.mr.topicFactory|:org.onap.dmaap.mr.topic: +enforced.topic.name.AAF=org.onap.dmaap.mr +forceAAF=false +transidUEBtopicreqd=false +defaultNSforUEB=org.onap.dmaap.mr +############################################################################## +#Mirror Maker Agent + +msgRtr.mirrormakeradmin.aaf=org.onap.dmaap.mr.mirrormaker|*|admin +msgRtr.mirrormakeruser.aaf=org.onap.dmaap.mr.mirrormaker|*|user +msgRtr.mirrormakeruser.aaf.create=org.onap.dmaap.mr.topicFactory|:org.onap.dmaap.mr.topic: +msgRtr.mirrormaker.timeout=15000 +msgRtr.mirrormaker.topic=org.onap.dmaap.mr.mirrormakeragent +msgRtr.mirrormaker.consumergroup=mmagentserver +msgRtr.mirrormaker.consumerid=1 + +kafka.max.poll.interval.ms=300000 +kafka.heartbeat.interval.ms=60000 +kafka.session.timeout.ms=240000 +kafka.max.poll.records=1000 \ No newline at end of file diff --git a/csit/scripts/dmaap-message-router/docker-compose/mr/cadi.properties b/csit/scripts/dmaap-message-router/docker-compose/mr/cadi.properties new file mode 100644 index 0000000..e7d056b --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/mr/cadi.properties @@ -0,0 +1,19 @@ +aaf_locate_url=https://aaf-onap-test.osaaf.org:8095\ +aaf_url=https://AAF_LOCATE_URL/onap.org.osaaf.aaf.service:2.1 +aaf_env=DEV +aaf_lur=org.onap.aaf.cadi.aaf.v2_0.AAFLurPerm + +cadi_truststore=/appl/dmaapMR1/etc/org.onap.dmaap.mr.trust.jks +cadi_truststore_password=8FyfX+ar;0$uZQ0h9*oXchNX + +cadi_keyfile=/appl/dmaapMR1/etc/org.onap.dmaap.mr.keyfile + +cadi_alias=dmaapmr@mr.dmaap.onap.org +cadi_keystore=/appl/dmaapMR1/etc/org.onap.dmaap.mr.p12 +cadi_keystore_password=GDQttV7)BlOvWMf6F7tz&cjy +cadi_x509_issuers=CN=intermediateCA_1, OU=OSAAF, O=ONAP, C=US:CN=intermediateCA_7, OU=OSAAF, O=ONAP, C=US:CN=intermediateCA_9, OU=OSAAF, O=ONAP, C=US + +cadi_loglevel=INFO +cadi_protocols=TLSv1.1,TLSv1.2 +cadi_latitude=37.78187 +cadi_longitude=-122.26147 \ No newline at end of file diff --git a/csit/scripts/dmaap-message-router/docker-compose/mr/logback.xml b/csit/scripts/dmaap-message-router/docker-compose/mr/logback.xml new file mode 100644 index 0000000..f02a2db --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/mr/logback.xml @@ -0,0 +1,208 @@ + + + + ${module.ajsc.namespace.name} + + + + + ERROR + ACCEPT + DENY + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n + + + + + + + INFO + ACCEPT + DENY + + + + + + + "%d [%thread] %-5level %logger{1024} - %msg%n" + + + + class="ch.qos.logback.core.ConsoleAppender"> + + ERROR + ACCEPT + DENY + + + "%d [%thread] %-5level %logger{1024} - %msg%n" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + "%d [%thread] %-5level %logger{1024} - %msg%n" + + + + + + + "%d [%thread] %-5level %logger{1024} - %msg%n" + + + + 1000 + 0 + + + + + + + + + + + 1000 + 0 + + + + + + + + + + + + + + + + diff --git a/csit/scripts/dmaap-message-router/docker-compose/zk/zk_server_jaas.conf b/csit/scripts/dmaap-message-router/docker-compose/zk/zk_server_jaas.conf new file mode 100644 index 0000000..3d2767f --- /dev/null +++ b/csit/scripts/dmaap-message-router/docker-compose/zk/zk_server_jaas.conf @@ -0,0 +1,4 @@ +Server { + org.apache.zookeeper.server.auth.DigestLoginModule required + user_kafka="kafka_secret"; +}; \ No newline at end of file diff --git a/csit/scripts/get-instance-ip.sh b/csit/scripts/get-instance-ip.sh new file mode 100755 index 0000000..a236c02 --- /dev/null +++ b/csit/scripts/get-instance-ip.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $1 diff --git a/csit/tests/mrpubsub/mrpubsub.robot b/csit/tests/mrpubsub/mrpubsub.robot new file mode 100755 index 0000000..355c367 --- /dev/null +++ b/csit/tests/mrpubsub/mrpubsub.robot @@ -0,0 +1,74 @@ +*** Settings *** +Library OperatingSystem +Library RequestsLibrary +Library requests +Library Collections +Library String +Library Process + +*** Variables *** +${TARGETURL_PUBLISH} http://${DMAAP_MR_IP}:3904/events/TestTopic1 +${TARGETURL_TOPICS} http://${DMAAP_MR_IP}:3904/topics +${TARGETURL_SUBSCR} http://${DMAAP_MR_IP}:3904/events/TestTopic1/CG1/C1?timeout=1000 +${TEST_DATA} {"topicName": "TestTopic1"} +${TOPIC_DATA} {"topicName":"FirstTopic","topicDescription":"This is a TestTopic","partitionCount":"1","replicationCount":"3","transactionEnabled":"true"} + +*** Test Cases *** +Run Topic Creation and Publish + [Documentation] Topic Creation + [Timeout] 1 minute + ${resp}= PostCall ${TARGETURL_PUBLISH} ${TEST_DATA} + log ${TARGETURL_PUBLISH} + log ${resp.text} + Should Be Equal As Strings ${resp.status_code} 200 + ${count}= Evaluate $resp.json().get('count') + log 'JSON Response Code:'${resp} + +Run Subscribing a message status + [Documentation] Subscribide message status + [Timeout] 1 minute + ${resp}= GetCall ${TARGETURL_SUBSCR} + log ${TARGETURL_SUBSCR} + Should Be Equal As Strings ${resp.status_code} 200 + log 'JSON Response Code :'${resp} + +Run check topics are exisiting + [Documentation] Get the count of the Topics + [Timeout] 1 minute + ${resp}= GetCall ${TARGETURL_TOPICS} + log ${TARGETURL_TOPICS} + Should Be Equal As Strings ${resp.status_code} 200 + log 'JSON Response Code :'${resp} + ${topics}= Evaluate $resp.json().get('topics') + log ${topics} + ${ListLength}= Get Length ${topics} + log ${ListLength} + List Should Contain Value ${topics} TestTopic1 + +Run Publish and Subscribe a message + [Documentation] Publish and Subscribe the message + [Timeout] 1 minute + ${resp}= PostCall ${TARGETURL_PUBLISH} ${TEST_DATA} + log ${TARGETURL_PUBLISH} + log ${resp.text} + Should Be Equal As Strings ${resp.status_code} 200 + ${sub_resp}= GetCall ${TARGETURL_SUBSCR} + log ${TARGETURL_SUBSCR} + Should Be Equal As Strings ${sub_resp.status_code} 200 + log 'JSON Response Code :'${sub_resp} + ${ListLength}= Get Length ${sub_resp.json()} + log ${ListLength} + List Should Contain Value ${sub_resp.json()} {"topicName":"TestTopic1"} case_insensitive=yes + +*** Keywords *** +PostCall + [Arguments] ${url} ${data} + ${headers}= Create Dictionary Accept=application/json Content-Type=application/json + ${resp}= Evaluate requests.post('${url}',data='${data}', headers=${headers},verify=False) requests + [Return] ${resp} + +GetCall + [Arguments] ${url} + ${headers}= Create Dictionary Accept=application/json Content-Type=application/json + ${resp}= Evaluate requests.get('${url}', headers=${headers}, verify=False) requests + [Return] ${resp} diff --git a/pom.xml b/pom.xml index 1ebf280..4eb9642 100644 --- a/pom.xml +++ b/pom.xml @@ -310,7 +310,7 @@ com.101tec zkclient - 0.2 + 0.11 org.slf4j @@ -755,7 +755,7 @@ docker - + true @@ -792,7 +792,7 @@ io.fabric8 docker-maven-plugin - 0.28.0 + 0.33.0 ${skip.docker.build} ${skip.docker.push} diff --git a/src/main/ajsc/dmaap_v1/dmaap/v1/conf/jaxrsBeans.groovy b/src/main/ajsc/dmaap_v1/dmaap/v1/conf/jaxrsBeans.groovy index 326b448..0841a3a 100644 --- a/src/main/ajsc/dmaap_v1/dmaap/v1/conf/jaxrsBeans.groovy +++ b/src/main/ajsc/dmaap_v1/dmaap/v1/conf/jaxrsBeans.groovy @@ -7,7 +7,7 @@ userService(org.onap.dmaap.JaxrsUserService) topicService(org.onap.dmaap.service.TopicRestService) eventService(org.onap.dmaap.service.EventsRestService) - adminServiceorg.(org.onap.dmaap.service.AdminRestService) + adminService(org.onap.dmaap.service.AdminRestService) apiKeyService(org.onap.dmaap.service.ApiKeysRestService) metricsService(org.onap.dmaap.service.MetricsRestService) transactionService(org.onap.dmaap.service.TransactionRestService) diff --git a/src/main/resources/docker-compose/docker-compose-old.yml b/src/main/resources/docker-compose/docker-compose-old.yml deleted file mode 100644 index d84e8de..0000000 --- a/src/main/resources/docker-compose/docker-compose-old.yml +++ /dev/null @@ -1,57 +0,0 @@ -version: '2' -services: - zookeeper: - image: nexus3.onap.org:10001/onap/dmaap/zookeeper:6.0.3 - ports: - - "2181:2181" - environment: - ZOOKEEPER_REPLICAS: 1 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_SYNC_LIMIT: 5 - ZOOKEEPER_INIT_LIMIT: 10 - ZOOKEEPER_MAX_CLIENT_CNXNS: 200 - ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT: 3 - ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: 24 - ZOOKEEPER_CLIENT_PORT: 2181 - KAFKA_OPTS: -Djava.security.auth.login.config=/etc/zookeeper/secrets/jaas/zk_server_jaas.conf -Dzookeeper.kerberos.removeHostFromPrincipal=true -Dzookeeper.kerberos.removeRealmFromPrincipal=true -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dzookeeper.requireClientAuthScheme=sasl - ZOOKEEPER_SERVER_ID: 1 - volumes: - - ./zk/zk_server_jaas.conf:/opt/app/subscriber/etc/subscriber.properties - - kafka: - image: nexus3.onap.org:10001/onap/dmaap/kafka111:1.0.4 - ports: - - "9092:9092" - environment: - enableCadi: 'false' - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 40000 - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 40000 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL_PLAINTEXT:PLAINTEXT,EXTERNAL_PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL_PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: INTERNAL_PLAINTEXT://0.0.0.0:9092 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL_PLAINTEXT - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/jaas/zk_client_jaas.conf - KAFKA_ZOOKEEPER_SET_ACL: 'true' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - depends_on: - - zookeeper - - dmaap: - image: nexus3.onap.org:10001/onap/dmaap/dmaap-mr:1.1.18 - ports: - - "3904:3904" - - "3905:3905" - environment: - enableCadi: 'false' - volumes: - - /var/tmp/MsgRtrApi.properties:/appl/dmaapMR1/bundleconfig/etc/appprops/MsgRtrApi.properties - depends_on: - - zookeeper - - kafka -networks: - net: - driver: bridge diff --git a/src/main/resources/docker-compose/docker-compose.yml b/src/main/resources/docker-compose/docker-compose.yml deleted file mode 100644 index 9cba0a2..0000000 --- a/src/main/resources/docker-compose/docker-compose.yml +++ /dev/null @@ -1,73 +0,0 @@ -version: '2' -services: - zookeeper: - image: nexus3.onap.org:10001/onap/dmaap/zookeeper:latest - ports: - - "2181:2181" - environment: - ZOOKEEPER_REPLICAS: 1 - ZOOKEEPER_TICK_TIME: 2000 - ZOOKEEPER_SYNC_LIMIT: 5 - ZOOKEEPER_INIT_LIMIT: 10 - ZOOKEEPER_MAX_CLIENT_CNXNS: 200 - ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT: 3 - ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: 24 - ZOOKEEPER_CLIENT_PORT: 2181 - KAFKA_OPTS: -Djava.security.auth.login.config=/etc/zookeeper/secrets/jaas/zk_server_jaas.conf -Dzookeeper.kerberos.removeHostFromPrincipal=true -Dzookeeper.kerberos.removeRealmFromPrincipal=true -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider -Dzookeeper.requireClientAuthScheme=sasl -Dzookeeper.4lw.commands.whitelist=* - ZOOKEEPER_SERVER_ID: 1 - volumes: - - ./zk/zk_server_jaas.conf:/etc/zookeeper/secrets/jaas/zk_server_jaas.conf - networks: - net: - aliases: - - zookeeper - - kafka: - image: nexus3.onap.org:10001/onap/dmaap/kafka111:latest - ports: - - "9092:9092" - environment: - enableCadi: 'false' - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 40000 - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: 40000 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL_PLAINTEXT:PLAINTEXT,EXTERNAL_PLAINTEXT:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL_PLAINTEXT://kafka:9092 - KAFKA_LISTENERS: INTERNAL_PLAINTEXT://0.0.0.0:9092 - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL_PLAINTEXT - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/jaas/zk_client_jaas.conf - KAFKA_ZOOKEEPER_SET_ACL: 'true' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - # Reduced the number of partitions only to avoid the timeout error for the first subscribe call in slow environment - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1 - volumes: - - ./kafka/zk_client_jaas.conf:/etc/kafka/secrets/jaas/zk_client_jaas.conf - networks: - net: - aliases: - - kafka - depends_on: - - zookeeper - - dmaap: - image: nexus3.onap.org:10001/onap/dmaap/dmaap-mr:latest - ports: - - "3904:3904" - - "3905:3905" - environment: - enableCadi: 'false' - volumes: - - ./mr/MsgRtrApi.properties:/appl/dmaapMR1/bundleconfig/etc/appprops/MsgRtrApi.properties - - ./mr/logback.xml:/appl/dmaapMR1/bundleconfig/etc/logback.xml - - ./mr/cadi.properties:/appl/dmaapMR1/etc/cadi.properties - networks: - net: - aliases: - - dmaap - depends_on: - - zookeeper - - kafka -networks: - net: - driver: bridge diff --git a/src/main/resources/docker-compose/kafka/zk_client_jaas.conf b/src/main/resources/docker-compose/kafka/zk_client_jaas.conf deleted file mode 100644 index 79a7601..0000000 --- a/src/main/resources/docker-compose/kafka/zk_client_jaas.conf +++ /dev/null @@ -1,5 +0,0 @@ -Client { - org.apache.zookeeper.server.auth.DigestLoginModule required - username="kafka" - password="kafka_secret"; - }; \ No newline at end of file diff --git a/src/main/resources/docker-compose/mr/MsgRtrApi.properties b/src/main/resources/docker-compose/mr/MsgRtrApi.properties deleted file mode 100644 index 8f8260e..0000000 --- a/src/main/resources/docker-compose/mr/MsgRtrApi.properties +++ /dev/null @@ -1,172 +0,0 @@ -# LICENSE_START======================================================= -# org.onap.dmaap -# ================================================================================ -# Copyright © 2017 AT&T Intellectual Property. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= -# -# ECOMP is a trademark and service mark of AT&T Intellectual Property. -# -############################################################################### -############################################################################### -## -## Cambria API Server config -## -## Default values are shown as commented settings. -## -############################################################################### -## -## HTTP service -## -## 3904 is standard as of 7/29/14. -# -## Zookeeper Connection -## -## Both Cambria and Kafka make use of Zookeeper. -## -#config.zk.servers=172.18.1.1 -#config.zk.servers={{.Values.zookeeper.name}}:{{.Values.zookeeper.port}} -config.zk.servers=zookeeper:2181 - -#config.zk.root=/fe3c/cambria/config - - -############################################################################### -## -## Kafka Connection -## -## Items below are passed through to Kafka's producer and consumer -## configurations (after removing "kafka.") -## if you want to change request.required.acks it can take this one value -#kafka.metadata.broker.list=localhost:9092,localhost:9093 -#kafka.metadata.broker.list={{.Values.kafka.name}}:{{.Values.kafka.port}} -kafka.metadata.broker.list=kafka:9092 -##kafka.request.required.acks=-1 -#kafka.client.zookeeper=${config.zk.servers} -consumer.timeout.ms=100 -zookeeper.connection.timeout.ms=6000 -zookeeper.session.timeout.ms=20000 -zookeeper.sync.time.ms=2000 -auto.commit.interval.ms=1000 -fetch.message.max.bytes =1000000 -auto.commit.enable=false - -#(backoff*retries > zksessiontimeout) -kafka.rebalance.backoff.ms=10000 -kafka.rebalance.max.retries=6 - - -############################################################################### -## -## Secured Config -## -## Some data stored in the config system is sensitive -- API keys and secrets, -## for example. to protect it, we use an encryption layer for this section -## of the config. -## -## The key is a base64 encode AES key. This must be created/configured for -## each installation. -#cambria.secureConfig.key= -## -## The initialization vector is a 16 byte value specific to the secured store. -## This must be created/configured for each installation. -#cambria.secureConfig.iv= - -## Southfield Sandbox -cambria.secureConfig.key=b/7ouTn9FfEw2PQwL0ov/Q== -cambria.secureConfig.iv=wR9xP5k5vbz/xD0LmtqQLw== -authentication.adminSecret=fe3cCompound -#cambria.secureConfig.key[pc569h]=YT3XPyxEmKCTLI2NK+Sjbw== -#cambria.secureConfig.iv[pc569h]=rMm2jhR3yVnU+u2V9Ugu3Q== - - -############################################################################### -## -## Consumer Caching -## -## Kafka expects live connections from the consumer to the broker, which -## obviously doesn't work over connectionless HTTP requests. The Cambria -## server proxies HTTP requests into Kafka consumer sessions that are kept -## around for later re-use. Not doing so is costly for setup per request, -## which would substantially impact a high volume consumer's performance. -## -## This complicates Cambria server failover, because we often need server -## A to close its connection before server B brings up the replacement. -## - -## The consumer cache is normally enabled. -#cambria.consumer.cache.enabled=true - -## Cached consumers are cleaned up after a period of disuse. The server inspects -## consumers every sweepFreqSeconds and will clean up any connections that are -## dormant for touchFreqMs. -#cambria.consumer.cache.sweepFreqSeconds=15 -cambria.consumer.cache.touchFreqMs=120000 -##stickforallconsumerrequests=false -## The cache is managed through ZK. The default value for the ZK connection -## string is the same as config.zk.servers. -#cambria.consumer.cache.zkConnect=${config.zk.servers} - -## -## Shared cache information is associated with this node's name. The default -## name is the hostname plus the HTTP service port this host runs on. (The -## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(), -## which is not always adequate.) You can set this value explicitly here. -## -#cambria.api.node.identifier= - -#cambria.rateLimit.maxEmptyPollsPerMinute=30 -#cambria.rateLimitActual.delay.ms=10 - -############################################################################### -## -## Metrics Reporting -## -## This server can report its metrics periodically on a topic. -## -#metrics.send.cambria.enabled=true -#metrics.send.cambria.topic=cambria.apinode.metrics #msgrtr.apinode.metrics.dmaap -#metrics.send.cambria.sendEverySeconds=60 - -cambria.consumer.cache.zkBasePath=/fe3c/cambria/consumerCache -consumer.timeout=17 -default.partitions=3 -default.replicas=3 -############################################################################## -#100mb -maxcontentlength=10000 - - -############################################################################## -#AAF Properties -msgRtr.namespace.aaf=org.onap.dmaap.mr.topic -msgRtr.topicfactory.aaf=org.onap.dmaap.mr.topicFactory|:org.onap.dmaap.mr.topic: -enforced.topic.name.AAF=org.onap.dmaap.mr -forceAAF=false -transidUEBtopicreqd=false -defaultNSforUEB=org.onap.dmaap.mr -############################################################################## -#Mirror Maker Agent - -msgRtr.mirrormakeradmin.aaf=org.onap.dmaap.mr.mirrormaker|*|admin -msgRtr.mirrormakeruser.aaf=org.onap.dmaap.mr.mirrormaker|*|user -msgRtr.mirrormakeruser.aaf.create=org.onap.dmaap.mr.topicFactory|:org.onap.dmaap.mr.topic: -msgRtr.mirrormaker.timeout=15000 -msgRtr.mirrormaker.topic=org.onap.dmaap.mr.mirrormakeragent -msgRtr.mirrormaker.consumergroup=mmagentserver -msgRtr.mirrormaker.consumerid=1 - -kafka.max.poll.interval.ms=300000 -kafka.heartbeat.interval.ms=60000 -kafka.session.timeout.ms=240000 -kafka.max.poll.records=1000 \ No newline at end of file diff --git a/src/main/resources/docker-compose/mr/cadi.properties b/src/main/resources/docker-compose/mr/cadi.properties deleted file mode 100644 index e7d056b..0000000 --- a/src/main/resources/docker-compose/mr/cadi.properties +++ /dev/null @@ -1,19 +0,0 @@ -aaf_locate_url=https://aaf-onap-test.osaaf.org:8095\ -aaf_url=https://AAF_LOCATE_URL/onap.org.osaaf.aaf.service:2.1 -aaf_env=DEV -aaf_lur=org.onap.aaf.cadi.aaf.v2_0.AAFLurPerm - -cadi_truststore=/appl/dmaapMR1/etc/org.onap.dmaap.mr.trust.jks -cadi_truststore_password=8FyfX+ar;0$uZQ0h9*oXchNX - -cadi_keyfile=/appl/dmaapMR1/etc/org.onap.dmaap.mr.keyfile - -cadi_alias=dmaapmr@mr.dmaap.onap.org -cadi_keystore=/appl/dmaapMR1/etc/org.onap.dmaap.mr.p12 -cadi_keystore_password=GDQttV7)BlOvWMf6F7tz&cjy -cadi_x509_issuers=CN=intermediateCA_1, OU=OSAAF, O=ONAP, C=US:CN=intermediateCA_7, OU=OSAAF, O=ONAP, C=US:CN=intermediateCA_9, OU=OSAAF, O=ONAP, C=US - -cadi_loglevel=INFO -cadi_protocols=TLSv1.1,TLSv1.2 -cadi_latitude=37.78187 -cadi_longitude=-122.26147 \ No newline at end of file diff --git a/src/main/resources/docker-compose/mr/logback.xml b/src/main/resources/docker-compose/mr/logback.xml deleted file mode 100644 index f02a2db..0000000 --- a/src/main/resources/docker-compose/mr/logback.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - ${module.ajsc.namespace.name} - - - - - ERROR - ACCEPT - DENY - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n - - - - - - - INFO - ACCEPT - DENY - - - - - - - "%d [%thread] %-5level %logger{1024} - %msg%n" - - - - class="ch.qos.logback.core.ConsoleAppender"> - - ERROR - ACCEPT - DENY - - - "%d [%thread] %-5level %logger{1024} - %msg%n" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "%d [%thread] %-5level %logger{1024} - %msg%n" - - - - - - - "%d [%thread] %-5level %logger{1024} - %msg%n" - - - - 1000 - 0 - - - - - - - - - - - 1000 - 0 - - - - - - - - - - - - - - - - diff --git a/src/main/resources/docker-compose/zk/zk_server_jaas.conf b/src/main/resources/docker-compose/zk/zk_server_jaas.conf deleted file mode 100644 index 3d2767f..0000000 --- a/src/main/resources/docker-compose/zk/zk_server_jaas.conf +++ /dev/null @@ -1,4 +0,0 @@ -Server { - org.apache.zookeeper.server.auth.DigestLoginModule required - user_kafka="kafka_secret"; -}; \ No newline at end of file -- cgit 1.2.3-korg