From b048fb63e8d671c9219071821deedb1f1ddf9f85 Mon Sep 17 00:00:00 2001 From: Lasse Kaihlavirta Date: Thu, 5 Nov 2020 18:01:29 +0200 Subject: Initial test commit for project-specific CSIT - move music-distributed-kv-store CSIT to its own project repository - add execution script that copies required common run scripts from integration/csit repo to local workspace and runs the tests - add the possibility to provide additional docker tag as parameter to the build script - add script for tagging specified docker image with STAGING Issue-ID: INT-1769 Signed-off-by: Lasse Kaihlavirta Change-Id: I19a5908f3c9c77529c43af0d89baf75a8f8557ee --- csit/.gitignore | 5 + .../music-distributed-kv-store-test-plan/setup.sh | 55 ++++++ .../teardown.sh | 21 +++ .../testplan.txt | 3 + csit/run-csit.sh | 199 +++++++++++++++++++++ csit/run-project-csit.sh | 42 +++++ .../__init__.robot | 2 + .../music-distributed-kv-store-test.robot | 53 ++++++ deployment/docker-build.sh | 9 +- deployment/tag-docker-staging.sh | 19 ++ 10 files changed, 406 insertions(+), 2 deletions(-) create mode 100644 csit/.gitignore create mode 100644 csit/plans/music-distributed-kv-store-test-plan/setup.sh create mode 100644 csit/plans/music-distributed-kv-store-test-plan/teardown.sh create mode 100644 csit/plans/music-distributed-kv-store-test-plan/testplan.txt create mode 100755 csit/run-csit.sh create mode 100755 csit/run-project-csit.sh create mode 100644 csit/tests/music-distributed-kv-store-suite/__init__.robot create mode 100644 csit/tests/music-distributed-kv-store-suite/music-distributed-kv-store-test.robot create mode 100755 deployment/tag-docker-staging.sh diff --git a/csit/.gitignore b/csit/.gitignore new file mode 100644 index 0000000..2b9c92f --- /dev/null +++ b/csit/.gitignore @@ -0,0 +1,5 @@ +run-csit.sh +prepare-csit.sh +env.properties +data/ +archives/ diff --git a/csit/plans/music-distributed-kv-store-test-plan/setup.sh b/csit/plans/music-distributed-kv-store-test-plan/setup.sh new file mode 100644 index 0000000..ccfe93b --- /dev/null +++ b/csit/plans/music-distributed-kv-store-test-plan/setup.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# +# Copyright 2020 Samsung Electronics etc. etc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Initial Configuration. +DATASTORE="consul" +DATASTORE_IP="localhost" + +MOUNTPATH="/dkv_mount_path/configs/" +DEFAULT_CONFIGS=$(pwd)/mountpath/default +TAG=1.0-SNAPSHOT-${UNIQUE_DOCKER_TAG} + +mkdir -p mountpath/default + +pushd mountpath/default +cat << EOF > sampleConfig1.properties +foo1=bar1 +hello1=world1 +key1=value1 +EOF +cat << EOF > sampleConfig2.properties +foo2=bar2 +hello2=world2 +key2=value2 +EOF +popd + +docker login -u docker -p docker nexus3.onap.org:10001 +docker pull nexus3.onap.org:10001/onap/music/distributed-kv-store:${TAG} +docker run -e DATASTORE=$DATASTORE -e DATASTORE_IP=$DATASTORE_IP -e MOUNTPATH=$MOUNTPATH -d \ + --name dkv \ + -v $DEFAULT_CONFIGS:/dkv_mount_path/configs/default \ + -p 8200:8200 -p 8080:8080 nexus3.onap.org:10001/onap/music/distributed-kv-store:${TAG} + +sleep 10 +# +# add here all ROBOT_VARIABLES settings +# +echo "# music robot variables settings"; +ROBOT_VARIABLES="-v DKV_HOSTNAME:http://localhost -v DKV_PORT:8080" + +echo ${ROBOT_VARIABLES} diff --git a/csit/plans/music-distributed-kv-store-test-plan/teardown.sh b/csit/plans/music-distributed-kv-store-test-plan/teardown.sh new file mode 100644 index 0000000..0abf3a6 --- /dev/null +++ b/csit/plans/music-distributed-kv-store-test-plan/teardown.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +rm -rf mountpath +docker stop dkv +docker rm dkv +docker rmi nexus3.onap.org:10001/onap/music/distributed-kv-store diff --git a/csit/plans/music-distributed-kv-store-test-plan/testplan.txt b/csit/plans/music-distributed-kv-store-test-plan/testplan.txt new file mode 100644 index 0000000..184b221 --- /dev/null +++ b/csit/plans/music-distributed-kv-store-test-plan/testplan.txt @@ -0,0 +1,3 @@ +# Test suites are relative paths under csit/tests/. +# Place the suites in run order. +music-distributed-kv-store-suite diff --git a/csit/run-csit.sh b/csit/run-csit.sh new file mode 100755 index 0000000..1aaf3e9 --- /dev/null +++ b/csit/run-csit.sh @@ -0,0 +1,199 @@ +#!/bin/bash -x +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# Modification Copyright 2019 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# $1 project/functionality +# $2 robot options + +# +# functions +# + +function on_exit(){ + rc=$? + if [[ ${WORKSPACE} ]]; then + if [[ ${WORKDIR} ]]; then + rsync -av "$WORKDIR/" "$WORKSPACE/archives/$TESTPLAN" + fi + # Record list of active docker containers + docker ps --format "{{.Image}}" > "$WORKSPACE/archives/$TESTPLAN/_docker-images.log" + + # show memory consumption after all docker instances initialized + docker_stats | tee "$WORKSPACE/archives/$TESTPLAN/_sysinfo-2-after-robot.txt" + fi + + # hack to test multiple results in Jenkins + cp -r "$WORKSPACE/archives/$TESTPLAN" "$WORKSPACE/archives/${TESTPLAN}_2" + + # Run teardown script plan if it exists + cd "${TESTPLANDIR}" + TEARDOWN="${TESTPLANDIR}/teardown.sh" + if [ -f "${TEARDOWN}" ]; then + echo "Running teardown script ${TEARDOWN}" + source_safely "${TEARDOWN}" + fi + # TODO: do something with the output + exit $rc +} +# ensure that teardown and other finalizing steps are always executed +trap on_exit EXIT + +function docker_stats(){ + #General memory details + echo "> top -bn1 | head -3" + top -bn1 | head -3 + echo + + echo "> free -h" + free -h + echo + + #Memory details per Docker + echo "> docker ps" + docker ps + echo + + echo "> docker stats --no-stream" + docker stats --no-stream + echo +} + +# save current set options +function save_set() { + RUN_CSIT_SAVE_SET="$-" + RUN_CSIT_SHELLOPTS="$SHELLOPTS" +} + +# load the saved set options +function load_set() { + _setopts="$-" + + # bash shellopts + for i in $(echo "$SHELLOPTS" | tr ':' ' ') ; do + set +o ${i} + done + for i in $(echo "$RUN_CSIT_SHELLOPTS" | tr ':' ' ') ; do + set -o ${i} + done + + # other options + for i in $(echo "$_setopts" | sed 's/./& /g') ; do + set +${i} + done + set -${RUN_CSIT_SAVE_SET} +} + +# set options for quick bailout when error +function harden_set() { + set -xeo pipefail + set +u # enabled it would probably fail too many often +} + +# relax set options so the sourced file will not fail +# the responsibility is shifted to the sourced file... +function relax_set() { + set +e + set +o pipefail +} + +# wrapper for sourcing a file +function source_safely() { + [ -z "$1" ] && return 1 + relax_set + . "$1" + load_set +} + +# +# main +# + +# set and save options for quick failure +harden_set && save_set + +if [ $# -eq 0 ] +then + echo + echo "Usage: $0 plans// []" + echo + echo " , , : " + echo " The same values as for the '{project}-csit-{functionality}' JJB job template." + echo + exit 1 +fi + +if [ -z "$WORKSPACE" ]; then + export WORKSPACE=$(git rev-parse --show-toplevel) +fi + +if [ -f "${WORKSPACE}/${1}/testplan.txt" ]; then + export TESTPLAN="${1}" +else + echo "testplan not found: ${WORKSPACE}/${TESTPLAN}/testplan.txt" + exit 2 +fi + +export TESTOPTIONS="${2}" + +rm -rf "$WORKSPACE/archives/$TESTPLAN" +mkdir -p "$WORKSPACE/archives/$TESTPLAN" + +TESTPLANDIR="${WORKSPACE}/${TESTPLAN}" + +# Run installation of prerequired libraries +source_safely "${WORKSPACE}/prepare-csit.sh" + +# Activate the virtualenv containing all the required libraries installed by prepare-csit.sh +source_safely "${ROBOT_VENV}/bin/activate" + +WORKDIR=$(mktemp -d --suffix=-robot-workdir) +cd "${WORKDIR}" + +# Add csit scripts to PATH +export PATH="${PATH}:${WORKSPACE}/docker/scripts:${WORKSPACE}/scripts:${ROBOT_VENV}/bin" +export SCRIPTS="${WORKSPACE}/scripts" +export ROBOT_VARIABLES= + +# Sign in to nexus3 docker repo +docker login -u docker -p docker nexus3.onap.org:10001 + +# Run setup script plan if it exists +cd "${TESTPLANDIR}" +SETUP="${TESTPLANDIR}/setup.sh" +if [ -f "${SETUP}" ]; then + echo "Running setup script ${SETUP}" + source_safely "${SETUP}" +fi + +# show memory consumption after all docker instances initialized +docker_stats | tee "$WORKSPACE/archives/$TESTPLAN/_sysinfo-1-after-setup.txt" + +# Run test plan +cd "$WORKDIR" +echo "Reading the testplan:" +cat "${TESTPLANDIR}/testplan.txt" | egrep -v '(^[[:space:]]*#|^[[:space:]]*$)' | sed "s|^|${WORKSPACE}/tests/|" > testplan.txt +cat testplan.txt +SUITES=$( xargs -a testplan.txt ) + +echo ROBOT_VARIABLES="${ROBOT_VARIABLES}" +echo "Starting Robot test suites ${SUITES} ..." +relax_set +python -m robot.run -N ${TESTPLAN} -v WORKSPACE:/tmp ${ROBOT_VARIABLES} ${TESTOPTIONS} ${SUITES} +RESULT=$? +load_set +echo "RESULT: $RESULT" +# Note that the final steps are done in on_exit function after this exit! +exit $RESULT diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh new file mode 100755 index 0000000..1ecc6cd --- /dev/null +++ b/csit/run-project-csit.sh @@ -0,0 +1,42 @@ +#!/bin/bash -x +# +# Copyright 2020 © Samsung Electronics Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# $1 test options (passed on to run-csit.sh as such) +# +# UNIQUE_DOCKER_TAG environment variable is expected to be used +# by all test plans. If it is not given, default "latest" is used +# +set -x +export TESTOPTIONS=${1} +export UNIQUE_DOCKER_TAG=${UNIQUE_DOCKER_TAG:-latest} +export GERRIT_BRANCH=${GERRIT_BRANCH:-master} +export WORKSPACE=$(git rev-parse --show-toplevel)/csit +rm -rf ${WORKSPACE}/archives +mkdir -p ${WORKSPACE}/archives + +rm -rf ${WORKSPACE}/data + +mkdir -p ${WORKSPACE}/data +cd ${WORKSPACE}/data + +git clone https://gerrit.onap.org/r/integration/csit -b ${GERRIT_BRANCH} +cp csit/*.sh ${WORKSPACE}/ +cd ${WORKSPACE} +for dir in plans/*/ +do + dir=${dir%*/} # remove the trailing / + ./run-csit.sh ${dir} ${TESTOPTIONS} +done diff --git a/csit/tests/music-distributed-kv-store-suite/__init__.robot b/csit/tests/music-distributed-kv-store-suite/__init__.robot new file mode 100644 index 0000000..d1da7f3 --- /dev/null +++ b/csit/tests/music-distributed-kv-store-suite/__init__.robot @@ -0,0 +1,2 @@ +*** Settings *** +Documentation Integration - Suite 1 \ No newline at end of file diff --git a/csit/tests/music-distributed-kv-store-suite/music-distributed-kv-store-test.robot b/csit/tests/music-distributed-kv-store-suite/music-distributed-kv-store-test.robot new file mode 100644 index 0000000..de26e5f --- /dev/null +++ b/csit/tests/music-distributed-kv-store-suite/music-distributed-kv-store-test.robot @@ -0,0 +1,53 @@ +*** Settings *** +Library OperatingSystem +Library RequestsLibrary +Library json + +*** Variables *** +${MESSAGE} {"ping": "ok"} + +#global variables +${generatedAID} + +*** Test Cases *** +DKV Check Distributed KV Store API Docker Container + [Documentation] Checks if DKV docker container is running + ${rc} ${output}= Run and Return RC and Output docker ps + Log To Console ********************* + Log To Console retrurn_code = ${rc} + Log To Console output = ${output} + Should Be Equal As Integers ${rc} 0 + Should Contain ${output} nexus3.onap.org:10001/onap/music/distributed-kv-store + +DKV LoadDefaultProperties + [Documentation] Loads default configuration files into Consul + Create Session dkv ${DKV_HOSTNAME}:${DKV_PORT} + &{headers}= Create Dictionary Content-Type=application/json Accept=application/json + ${resp}= Get Request dkv /v1/config/load-default headers=${headers} + Log To Console ********************* + Log To Console response = ${resp} + Log To Console body = ${resp.text} + Should Be Equal As Integers ${resp.status_code} 200 + +DKV FetchDefaultProperties + [Documentation] Fetches all default keys from Consul + Create Session dkv ${DKV_HOSTNAME}:${DKV_PORT} + &{headers}= Create Dictionary Content-Type=application/json Accept=application/json + ${resp}= Get Request dkv /v1/getconfigs headers=${headers} + Log To Console ********************* + Log To Console response = ${resp} + Log To Console body = ${resp.text} + Should Be Equal As Integers ${resp.status_code} 200 + +#DKV RegisterDomain +# [Documentation] Send a POST request to create a domain +# Create Session dkv ${DKV_HOSTNAME}:${DKV_PORT} +# ${data}= Get Binary File ${CURDIR}${/}data${/}register_domain.json +# &{headers}= Create Dictionary Content-Type=application/json Accept=application/json +# ${resp}= Post Request dkv v1/register data=${data} headers=${headers} +# Log To Console ********************* +# Log To Console response = ${resp} +# Log To Console body = ${resp.text} +# Should Be Equal As Integers ${resp.status_code} 200 + +*** Keywords *** diff --git a/deployment/docker-build.sh b/deployment/docker-build.sh index 4568e53..c8f3e1e 100755 --- a/deployment/docker-build.sh +++ b/deployment/docker-build.sh @@ -2,13 +2,14 @@ BUILD_ARGS="--no-cache" ORG="onap" -VERSION="1.0.0" PROJECT="music" IMAGE="distributed-kv-store" DOCKER_REPOSITORY="nexus3.onap.org:10003" IMAGE_NAME="${DOCKER_REPOSITORY}/${ORG}/${PROJECT}/${IMAGE}" -TIMESTAMP=$(date +"%Y%m%dT%H%M%S") +TAG_NAME=${UNIQUE_DOCKER_TAG} +echo TAG_NAME: ${TAG_NAME} +set -x if [ $HTTP_PROXY ]; then BUILD_ARGS+=" --build-arg HTTP_PROXY=${HTTP_PROXY}" fi @@ -34,6 +35,10 @@ function build_image { function push_image { echo "Start push docker image." docker push ${IMAGE_NAME}:latest + if [ ! -z "${TAG_NAME}" ]; then + docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:1.0-SNAPSHOT-${TAG_NAME} + docker push ${IMAGE_NAME}:1.0-SNAPSHOT-${TAG_NAME} + fi } function remove_binary { diff --git a/deployment/tag-docker-staging.sh b/deployment/tag-docker-staging.sh new file mode 100755 index 0000000..66af8cb --- /dev/null +++ b/deployment/tag-docker-staging.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +ORG="onap" +PROJECT="music" +IMAGE="distributed-kv-store" +DOCKER_REPOSITORY="nexus3.onap.org:10003" +IMAGE_NAME="${DOCKER_REPOSITORY}/${ORG}/${PROJECT}/${IMAGE}" +TAG_NAME=${UNIQUE_DOCKER_TAG} + +echo TAG_NAME: ${TAG_NAME} +set -x + +echo "Push STAGING tag for docker image." +if [ ! -z "${TAG_NAME}" ]; then + docker pull ${IMAGE_NAME}:1.0-SNAPSHOT-${TAG_NAME} + docker tag ${IMAGE_NAME}:1.0-SNAPSHOT-${TAG_NAME} ${IMAGE_NAME}:1.0-STAGING-latest + docker push ${IMAGE_NAME}:1.0-STAGING-latest +fi + -- cgit 1.2.3-korg