From 1f7c861c65aa8689f11196581b8ca5e79d7e4e26 Mon Sep 17 00:00:00 2001 From: liamfallon Date: Tue, 20 Nov 2018 15:59:50 +0000 Subject: Refactor Apex benchmark tests Apex performance tests are refactored and reorganised to be all in the one place and to be coherent. A test REST server that can generate events in batches is introduced and this can be configured to respond to REST calls from Apex with various batch sizes that can be varied for performance tests. Actual performance tests are added for the various executors in Apex. Change-Id: I692b41e1e4451e821a618c64dead5cc8e9685d08 Issue-ID: POLICY-812 Signed-off-by: liamfallon --- .../src/main/package/scripts/apexApps.bat | 3 +- .../src/main/package/scripts/apexApps.sh | 2 + .../src/main/package/scripts/runBenchmark.sh | 74 +++++++++ .../src/main/package/scripts/runOneBenchmark.sh | 89 +++++++++++ .../src/main/resources/hazelcast/hazelcast.xml | 178 ++++++++++----------- .../src/main/resources/logback.xml | 2 +- 6 files changed, 257 insertions(+), 91 deletions(-) create mode 100644 packages/apex-pdp-package-full/src/main/package/scripts/runBenchmark.sh create mode 100755 packages/apex-pdp-package-full/src/main/package/scripts/runOneBenchmark.sh (limited to 'packages/apex-pdp-package-full/src') diff --git a/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.bat b/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.bat index d5ac0532c..848498b1e 100644 --- a/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.bat +++ b/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.bat @@ -81,6 +81,7 @@ set APEX_APP_MAP[engine]=java -Dlogback.configurationFile=%APEX_HOME%\etc\logbac set APEX_APP_MAP[eng-deployment]=java -Dlogback.configurationFile=%APEX_HOME%\etc\logback.xml -jar %APEX_HOME%\lib\applications\client-deployment-%_VERSION%-deployment.jar set APEX_APP_MAP[eng-monitoring]=java -Dlogback.configurationFile=%APEX_HOME%\etc\logback.xml -jar %APEX_HOME%\lib\applications\client-monitoring-%_VERSION%-monitoring.jar set APEX_APP_MAP[full-client]=java -Dlogback.configurationFile=%APEX_HOME%\etc\logback.xml -jar %APEX_HOME%\lib\applications\client-full-%_VERSION%-full.jar +set APEX_APP_MAP[event-generator]=java -Dlogback.configurationFile=%APEX_HOME%\etc\logback.xml -cp %CLASSPATH% %_CONFIG% org.onap.policy.apex.testsuites.performance.benchmark.eventgenerator.EventGenerator :: array of applications with name=description :: declare -A APEX_APP_DESCR_MAP @@ -94,7 +95,7 @@ set APEX_APP_DESCR_MAP[engine]=starts the APEX engine set APEX_APP_DESCR_MAP[eng-deployment]=starts the APEX deployment client in a simple webserver set APEX_APP_DESCR_MAP[eng-monitoring]=starts the APEX engine monitoring client in a simple webserver set APEX_APP_DESCR_MAP[full-client]=starts the full APEX client (rest editor, deployment, monitoring) in a simple webserver - +set APEX_APP_DESCR_MAP["event-generator"]=starts the event generator in a simple webserver for performance testing :: no command line means help, -h means help if "%1" == "" goto Help diff --git a/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.sh b/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.sh index 3316e8e1c..c5569baaa 100755 --- a/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.sh +++ b/packages/apex-pdp-package-full/src/main/package/scripts/apexApps.sh @@ -90,6 +90,7 @@ APEX_APP_MAP["engine"]="java -Dlogback.configurationFile=$APEX_HOME/etc/logback. APEX_APP_MAP["eng-deployment"]="java -Dlogback.configurationFile=$APEX_HOME/etc/logback.xml -jar $APEX_HOME/lib/applications/client-deployment-$_version-deployment.jar" APEX_APP_MAP["eng-monitoring"]="java -Dlogback.configurationFile=$APEX_HOME/etc/logback.xml -jar $APEX_HOME/lib/applications/client-monitoring-$_version-monitoring.jar" APEX_APP_MAP["full-client"]="java -Dlogback.configurationFile=$APEX_HOME/etc/logback.xml -jar $APEX_HOME/lib/applications/client-full-$_version-full.jar" +APEX_APP_MAP["event-gen"]="java -Dlogback.configurationFile=$APEX_HOME/etc/logback.xml -cp ${CLASSPATH} $_config org.onap.policy.apex.testsuites.performance.benchmark.eventgenerator.EventGenerator" ## array of applications with name=description declare -A APEX_APP_DESCR_MAP @@ -103,6 +104,7 @@ APEX_APP_DESCR_MAP["engine"]="starts the APEX engine" APEX_APP_DESCR_MAP["eng-deployment"]="starts the APEX deployment client in a simple webserver" APEX_APP_DESCR_MAP["eng-monitoring"]="starts the APEX engine monitoring client in a simple webserver" APEX_APP_DESCR_MAP["full-client"]="starts the full APEX client (rest editor, deployment, monitoring) in a simple webserver" +APEX_APP_DESCR_MAP["event-generator"]="starts the event generator in a simple webserver for performance testing" ## ## Help screen and exit condition (i.e. too few arguments) diff --git a/packages/apex-pdp-package-full/src/main/package/scripts/runBenchmark.sh b/packages/apex-pdp-package-full/src/main/package/scripts/runBenchmark.sh new file mode 100644 index 000000000..08c8d181f --- /dev/null +++ b/packages/apex-pdp-package-full/src/main/package/scripts/runBenchmark.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------- +# ============LICENSE_START======================================================= +# Copyright (C) 2018 Ericsson. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +#------------------------------------------------------------------------------- + +if [ -z $APEX_HOME ] +then + APEX_HOME="/opt/app/policy/apex-pdp" +fi + +if [ ! -d $APEX_HOME ] +then + echo + echo 'Apex directory "'$APEX_HOME'" not set or not a directory' + echo "Please set environment for 'APEX_HOME'" + exit +fi + +$APEX_HOME/bin/runOneBenchmark.sh Javascript 01 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 02 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 04 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 08 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 16 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 32 +$APEX_HOME/bin/runOneBenchmark.sh Javascript 64 + +$APEX_HOME/bin/runOneBenchmark.sh Jython 01 +$APEX_HOME/bin/runOneBenchmark.sh Jython 02 +$APEX_HOME/bin/runOneBenchmark.sh Jython 04 +$APEX_HOME/bin/runOneBenchmark.sh Jython 08 +$APEX_HOME/bin/runOneBenchmark.sh Jython 16 +$APEX_HOME/bin/runOneBenchmark.sh Jython 32 +$APEX_HOME/bin/runOneBenchmark.sh Jython 64 + +$APEX_HOME/bin/runOneBenchmark.sh Mvel 01 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 02 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 04 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 08 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 16 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 32 +$APEX_HOME/bin/runOneBenchmark.sh Mvel 64 + +$APEX_HOME/bin/runOneBenchmark.sh Java 01 +$APEX_HOME/bin/runOneBenchmark.sh Java 02 +$APEX_HOME/bin/runOneBenchmark.sh Java 04 +$APEX_HOME/bin/runOneBenchmark.sh Java 08 +$APEX_HOME/bin/runOneBenchmark.sh Java 16 +$APEX_HOME/bin/runOneBenchmark.sh Java 32 +$APEX_HOME/bin/runOneBenchmark.sh Java 64 + +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 01 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 02 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 04 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 08 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 16 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 32 +#$APEX_HOME/bin/runOneBenchmark.sh JRuby 64 diff --git a/packages/apex-pdp-package-full/src/main/package/scripts/runOneBenchmark.sh b/packages/apex-pdp-package-full/src/main/package/scripts/runOneBenchmark.sh new file mode 100755 index 000000000..2e81bd29d --- /dev/null +++ b/packages/apex-pdp-package-full/src/main/package/scripts/runOneBenchmark.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +#------------------------------------------------------------------------------- +# ============LICENSE_START======================================================= +# Copyright (C) 2018 Ericsson. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +#------------------------------------------------------------------------------- + +if [ -z $APEX_HOME ] +then + APEX_HOME="/opt/app/policy/apex-pdp" +fi + +if [ ! -d $APEX_HOME ] +then + echo + echo 'Apex directory "'$APEX_HOME'" not set or not a directory' + echo "Please set environment for 'APEX_HOME'" + exit +fi + +if [ $# -ne 2 ] +then + echo "usage: $0 executor-type thread-count" + echo " executor-type [Javascript|Jython|JRuby|Mvel|Java]" + echo " thread-count [01|02|04|08|16|32|64]" + exit 1 +fi + +if [ "$1" != "Javascript" ] && [ "$1" != "Jython" ] && [ "$1" != "JRuby" ] && [ "$1" != "Mvel" ] && [ "$1" != "Java" ] +then + echo "executor-type must be a member of the set [Javascript|Jython|JRuby|Mvel|Java]" + exit 1 +fi + +if [ "$2" != "01" ] && [ "$2" != "02" ] && [ "$2" != "04" ] && [ "$2" != "08" ] && [ "$2" != "16" ] && [ "$2" != "32" ] && [ "$2" != "64" ] +then + echo "thread-count must be a member of the set [01|02|04|08|16|32|64]" + exit 1 +fi + +# Remove the old benchmark test result file if it exists +rm -fr examples/benchmark/Bm$1$2.json + +# Start the event generator +/bin/bash bin/apexApps.sh event-gen -c examples/benchmark/EventGeneratorConfig.json -o examples/benchmark/Bm$1$2.json > examples/benchmark/Bm$1$2_gen.log 2>&1 & + +# Start Apex +sleep 2 +/bin/bash bin/apexApps.sh engine -c examples/benchmark/$1$2.json > examples/benchmark/Bm$1$2_apex.log 2>&1 & +apex_pid=`ps -A -o pid,cmd | grep ApexMain | grep -v grep | head -n 1 | awk '{print $1}'` + +echo "running benchmark test for executor $1 with $2 threads" + +# Loop until result file exists +while [ ! -f examples/benchmark/Bm$1$2.json ] +do + echo -n . + sleep 1 +done + +# Loop until result file has 318 lines +while [ "$(wc -l examples/benchmark/Bm$1$2.json | cut -f1 -d' ')" -lt 318 ] +do + echo -n . + sleep 1 +done + +# Kill Apex +kill $apex_pid + +sleep 5 + +echo "" +echo "Finished" diff --git a/packages/apex-pdp-package-full/src/main/resources/hazelcast/hazelcast.xml b/packages/apex-pdp-package-full/src/main/resources/hazelcast/hazelcast.xml index f90c53c51..b0847f295 100644 --- a/packages/apex-pdp-package-full/src/main/resources/hazelcast/hazelcast.xml +++ b/packages/apex-pdp-package-full/src/main/resources/hazelcast/hazelcast.xml @@ -32,19 +32,19 @@ 10.0.0.1 - + - + 127.0.0.* true slf4j - + false 60 @@ -52,59 +52,59 @@ 16 - + 100000 - + - + 1 - + NONE - + 0 - + 25 - + 1 - + true - + LRU - + 3000000 - + 5 - + 7200 @@ -113,45 +113,45 @@ - + 1 - + true - + LRU - + 3000000 - + 5 - + 7200 - + 100000 - + 0 @@ -161,24 +161,24 @@ 3 true - + NONE - + 1000 - + 0 - + 0 @@ -186,34 +186,34 @@ 3 true - + NONE - + 1000 - + 0 - + 0 - + 100000 - + 1 NONE 0 diff --git a/packages/apex-pdp-package-full/src/main/resources/logback.xml b/packages/apex-pdp-package-full/src/main/resources/logback.xml index 42502b1ab..b96a37686 100644 --- a/packages/apex-pdp-package-full/src/main/resources/logback.xml +++ b/packages/apex-pdp-package-full/src/main/resources/logback.xml @@ -25,7 +25,7 @@ Apex - + %d %contextName [%t] %level %logger{36} - %msg%n -- cgit 1.2.3-korg