Add integration scripts from OPEN-O

Change-Id: Ife6951ed9ea8c5b9dcea68e7a095b6bd5180e7d1
Signed-off-by: Gary Wu <gary.i.wu@huawei.com>

5 files changed, 429 insertions, 0 deletions

diff --git a/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh
new file mode 100755
index 000000000..9432cee54
--- /dev/null
+++ b/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh
@@ -0,0 +1,191 @@
+#!/bin/bash
+#
+# Copyright 2016-2017 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# autorelease root dir
+ROOT=`git rev-parse --show-toplevel`/autorelease
+
+BUILD_DIR=$ROOT/build
+JJB_DIR=$BUILD_DIR/ci-management/jjb
+
+cd $BUILD_DIR
+
+source $ROOT/scripts/generate-jjbs/workarounds.sh
+
+
+find . -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do
+    project=${repo}
+ 
+    toxs=`find $repo -type d -exec test -e "{}/tox.ini" ';' -prune -printf "%P/tox.ini\n" | sort`
+
+    in_sun_branch=`git show sun:autorelease/all-projects.txt | grep -x ${repo}`
+
+    mkdir -p $JJB_DIR/$repo
+
+    if [ ! -z "$toxs" ]; then
+	rm -f $JJB_DIR/$repo/${repo}-python.yaml
+
+	project=${repo}-java
+	echo $repo/${repo}-python.yaml
+	
+	cat > $JJB_DIR/$repo/${repo}-python.yaml <<EOF
+---
+- project:
+    name: ${repo}-python
+    project: '${repo}'
+    stream:
+      - 'master':
+          branch: 'master'
+EOF
+	if [ $in_sun_branch ]; then
+	    cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF
+      - 'sun':
+          branch: 'sun'
+EOF
+	fi
+	cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF
+    mvn-settings: '${repo}-settings'
+    build-node: 'centos7-redis-2c-1g'
+    subproject:
+EOF
+	for tox in $toxs; do
+	    toxpath=${tox%/tox.ini}
+
+	    if [ "$toxpath" == "" ]; then
+		subproject="root"
+		pathparam="."
+		pattern="**"
+	    else
+		subproject=${toxpath////-} # replace slash with dash
+		pathparam=$toxpath
+		pattern="$toxpath/**"
+
+		# do special subproject names
+		for SUB in "${SUBPROJECT_MAP[@]}"; do
+		    if [ "${SUB%:*}" = "$repo/$toxpath" ]; then
+			subproject=${SUB#*:}
+		    fi
+		done
+	    fi
+
+	    cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF
+      - '${subproject}':
+          path: '${pathparam}'
+          pattern: '${pattern}'
+EOF
+	done
+	cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF
+    jobs:
+      - '{project}-{stream}-{subproject}-verify-python'
+EOF
+    fi
+
+
+
+
+    has_subprojects=0
+    for r in "${SPLIT_REPOS[@]}"; do
+	if [ "$repo" = "$r" ]; then
+	    has_subprojects=1
+	fi
+    done
+
+    if [ $has_subprojects -eq 1 ]; then
+	poms=`find $repo -mindepth 1 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort`
+    else
+	poms=`find $repo -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort`
+	if [ "$poms" != "/pom.xml" ]; then
+	    has_subprojects=1
+	fi
+    fi
+    
+    
+    if [ ! -z "$poms" ]; then
+	rm -f $JJB_DIR/$repo/${repo}.yaml
+	rm -f $JJB_DIR/$repo/${repo}-java.yaml
+	echo $repo/${project}.yaml
+    fi
+
+    if [ $has_subprojects -eq 0 ]; then
+	# root pom.xml found
+	cat > $JJB_DIR/$repo/${project}.yaml <<EOF
+---
+- project:
+    name: ${project}
+    jobs:
+      - '{project}-{stream}-verify-java'
+      - '{project}-{stream}-merge-java'
+
+    project: '${repo}'
+    stream:
+      - 'master':
+          branch: 'master'
+EOF
+	if [ $in_sun_branch ]; then
+	    cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+      - 'sun':
+          branch: 'sun'
+EOF
+	fi
+	cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+    mvn-settings: '${repo}-settings'
+EOF
+    elif [ ! -z "$poms" ]; then
+	cat > $JJB_DIR/$repo/${project}.yaml <<EOF
+---
+- project:
+    name: ${project}
+    project: '${repo}'
+    stream:
+      - 'master':
+          branch: 'master'
+EOF
+	if [ $in_sun_branch ]; then
+	    cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+      - 'sun':
+          branch: 'sun'
+EOF
+	fi
+	cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+    mvn-settings: '${repo}-settings'
+    subproject:
+EOF
+
+	for pom in $poms; do
+	    pompath=${pom%/pom.xml}
+	    subproject=${pompath////-} # replace slash with dash
+	    cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+      - '${subproject}':
+          pom: '${pom}'
+          pattern: '${pompath}/**'
+EOF
+	done
+
+	if [ -e $BUILD_DIR/$repo/pom.xml ]; then
+	    cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+      - 'root':
+          pom: 'pom.xml'
+          pattern: '*'
+EOF
+	fi
+
+	cat >> $JJB_DIR/$repo/${project}.yaml <<EOF
+    jobs:
+      - '{project}-{stream}-{subproject}-verify-java'
+      - '{project}-{stream}-{subproject}-merge-java'
+EOF
+    fi
+done
diff --git a/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh
new file mode 100755
index 000000000..539e91115
--- /dev/null
+++ b/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+#
+# Copyright 2016-2017 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# csit plans root dir
+ROOT=`git rev-parse --show-toplevel`/autorelease
+
+BUILD_DIR=$ROOT/build
+JJB_DIR=$BUILD_DIR/ci-management/jjb
+
+WORKSPACE=`git rev-parse --show-toplevel`
+PLANS_DIR=`git rev-parse --show-toplevel`/test/csit/plans
+
+source $ROOT/scripts/generate-jjbs/workarounds.sh
+
+
+TMPDIR=`mktemp -d --suffix="-docker-log"`
+$ROOT/scripts/ls-microservice-repos.py | cut -d ' ' -f 1 > $TMPDIR/microservices.txt
+
+find $PLANS_DIR -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do
+
+    OUTFILE=$JJB_DIR/$repo/${repo}-csit.yaml
+    cat > $OUTFILE <<EOF
+---
+- project:
+    name: ${repo}-csit
+    jobs:
+      - 'integration-verify-{project}-csit-{functionality}'
+      - '{project}-csit-{functionality}'
+    project: '${repo}'
+    functionality:
+EOF
+    find $PLANS_DIR/$repo -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read func; do
+	echo $repo / $func
+	cat >> $OUTFILE <<EOF
+      - '${func}':
+          trigger_jobs:
+EOF
+
+	docker kill `docker ps -a -q`
+	docker rm `docker ps -a -q`
+
+	$WORKSPACE/test/csit/run-csit.sh plans/${repo}/${func}
+	mkdir -p $TMPDIR/${repo}
+	cp $WORKSPACE/archives/_docker-images.log $TMPDIR/${repo}/${func}.txt
+	
+	docker kill `docker ps -a -q`
+	docker rm `docker ps -a -q`
+	
+	
+	for image in `grep openoint $TMPDIR/${repo}/${func}.txt | grep -f $TMPDIR/microservices.txt | sort`; do
+	    microservice=`echo $image | cut -d '/' -f 2`
+	    cat >> $OUTFILE <<EOF
+            - 'integration-${microservice}-merge-docker'
+EOF
+	done
+
+    done
+
+    cat >> $OUTFILE <<EOF
+    robot-options: ''
+    branch: 'master'
+EOF
+done
+
+echo $TMPDIR
diff --git a/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh
new file mode 100755
index 000000000..142892e72
--- /dev/null
+++ b/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Copyright 2016-2017 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# autorelease root dir
+ROOT=`git rev-parse --show-toplevel`/autorelease
+
+BUILD_DIR=$ROOT/build
+JJB_DIR=$BUILD_DIR/ci-management/jjb
+
+cd $BUILD_DIR
+
+# docker root dir
+DOCKER=`git rev-parse --show-toplevel`/test/csit/docker
+
+cd $DOCKER
+
+OUTFILE=$JJB_DIR/integration/integration-docker-microservices.yaml
+
+cat > $OUTFILE <<EOF
+---
+- project:
+    name: integration-docker-microservices
+    project: 'integration'
+    branch: 'master'
+    version: '1.1.0-SNAPSHOT'
+    mvn-settings: 'autorelease-settings'
+    build-node: 'centos7-robot-8c-8g'
+    jobs:
+      - 'integration-{microservice}-verify-docker'
+      - 'integration-{microservice}-merge-docker'
+    microservice:
+EOF
+
+TMPDIR=`$ROOT/scripts/generate-jjbs/gen-job-lists.sh`
+IFS=$'\n'
+for line in `$ROOT/scripts/ls-microservice-repos.py | sort`; do
+    IFS=' '
+    array=($line)
+    microservice=${array[0]}
+    repo=${array[1]}
+    trigger=${array[2]}
+    echo $microservice $repo $trigger
+    cat >> $OUTFILE <<EOF
+      - '${microservice}':
+          trigger_jobs:
+EOF
+    for job in `cat ${TMPDIR}/merge-jobs/${repo}.txt | grep "${trigger}"`; do
+    cat >> $OUTFILE <<EOF
+            - '${job}'
+EOF
+    done
+done
diff --git a/autorelease/scripts/generate-jjbs/gen-job-lists.sh b/autorelease/scripts/generate-jjbs/gen-job-lists.sh
new file mode 100755
index 000000000..4e12c91dc
--- /dev/null
+++ b/autorelease/scripts/generate-jjbs/gen-job-lists.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Copyright 2016-2017 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# autorelease root dir
+ROOT=`git rev-parse --show-toplevel`/autorelease
+
+BUILD_DIR=$ROOT/build
+JJB_DIR=$BUILD_DIR/ci-management/jjb
+
+cd $BUILD_DIR
+
+source $ROOT/scripts/generate-jjbs/workarounds.sh
+
+
+TMPDIR=`mktemp -d`
+echo $TMPDIR
+
+mkdir -p $TMPDIR/merge-jobs
+find . -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do
+    project=${repo}
+    OUTFILE=$TMPDIR/merge-jobs/${repo}.txt
+ 
+    has_subprojects=0
+    for r in "${SPLIT_REPOS[@]}"; do
+	if [ "$repo" = "$r" ]; then
+	    has_subprojects=1
+	fi
+    done
+
+    if [ $has_subprojects -eq 1 ]; then
+	poms=`find $repo -mindepth 1 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort`
+    else
+	poms=`find $repo -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort`
+	if [ "$poms" != "/pom.xml" ]; then
+	    has_subprojects=1
+	fi
+    fi
+    
+    
+    if [ $has_subprojects -eq 0 ]; then
+	echo ${repo}-master-merge-java > $OUTFILE
+    elif [ ! -z "$poms" ]; then
+	for pom in $poms; do
+	    pompath=${pom%/pom.xml}
+	    subproject=${pompath////-} # replace slash with dash
+	    echo ${repo}-master-${subproject}-merge-java >> $OUTFILE
+	done
+    fi
+done
+
diff --git a/autorelease/scripts/generate-jjbs/workarounds.sh b/autorelease/scripts/generate-jjbs/workarounds.sh
new file mode 100755
index 000000000..82b39658c
--- /dev/null
+++ b/autorelease/scripts/generate-jjbs/workarounds.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Copyright 2016-2017 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# specific workarounds to maintain existing job list
+rm -f $BUILD_DIR/integration/autorelease/build/pom.xml
+rm -f $BUILD_DIR/nfvo/monitor/pom.xml
+touch $BUILD_DIR/common-tosca-aria/tox.ini
+
+
+SUBPROJECT_MAP=(
+    "nfvo/drivers/vnfm/svnfm/zte/vmanager:drv-vnfm-zte"
+    "nfvo/drivers/vnfm/gvnfm/gvnfmadapter:drv-gvnfm"
+)
+
+SPLIT_REPOS=("nfvo" "common-services-common-utilities" "multivimdriver-openstack" "multivimdriver-vmware-vio")