diff options
Diffstat (limited to 'autorelease')
19 files changed, 1109 insertions, 0 deletions
diff --git a/autorelease/scripts/autorelease-test.sh b/autorelease/scripts/autorelease-test.sh new file mode 100755 index 000000000..b5d584e50 --- /dev/null +++ b/autorelease/scripts/autorelease-test.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease +GERRIT_BRANCH='master' + +cd $ROOT +git checkout build + +BUILD_DIR=$ROOT/build +cd $BUILD_DIR + +$ROOT/scripts/clone-all.sh +$ROOT/scripts/generate-pom.sh +$ROOT/scripts/fix-relativepaths.sh +$ROOT/scripts/set-version.sh + +TMPDIR=`mktemp -d` +mvn -q clean deploy -DdisableJavaSdkAutoGeneration -DaltDeploymentRepository=staging::default::file:$TMPDIR -DskipTests=true -Dcheckstyle.skip=true +echo "TMPDIR=$TMPDIR" diff --git a/autorelease/scripts/clone-all.sh b/autorelease/scripts/clone-all.sh new file mode 100755 index 000000000..936d8ea08 --- /dev/null +++ b/autorelease/scripts/clone-all.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease +GERRIT_BRANCH='master' + +BUILD_DIR=$ROOT/build +cd $BUILD_DIR + +$ROOT/scripts/get-all-repos.sh | while read p; do + cd $BUILD_DIR + if [ -e $BUILD_DIR/$p ]; then + cd $BUILD_DIR/$p + git checkout $GERRIT_BRANCH + git reset --hard origin + git clean -f + git pull + else + #TODO: replace with https once repo is open to public + git clone -b $GERRIT_BRANCH ssh://gerrit.open-o.org:29418/$p + fi +done + +rm -rf $BUILD_DIR/integration/autorelease/build diff --git a/autorelease/scripts/diff-all.sh b/autorelease/scripts/diff-all.sh new file mode 100755 index 000000000..5bb502f25 --- /dev/null +++ b/autorelease/scripts/diff-all.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease +GERRIT_BRANCH='sun' + +BUILD_DIR=$ROOT/build + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +$ROOT/scripts/get-all-repos.sh | while read p; do + cd $BUILD_DIR/$p + echo $p + git diff | cat +done diff --git a/autorelease/scripts/fix-names.sh b/autorelease/scripts/fix-names.sh new file mode 100755 index 000000000..9540ac594 --- /dev/null +++ b/autorelease/scripts/fix-names.sh @@ -0,0 +1,42 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015, 2016 The Linux Foundation. All rights reserved. +############################################################################## + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# MAP of path to a parent pom from the perspective of hosting directory +# starting from the autorelease repo root. +# +# Format: <groupId>:<artifactId>:<path> + +fix_name() { + pom=$1 + echo -e "\nScanning $pom" + pomPath=`dirname $pom` + + projectPath=${pomPath#*/} # Path to pom file from the perspective of hosting repo + + relativePath="$projectPath" # Calculated relative path to parent pom + + # Update any existing project names + xmlstarlet ed -P -N x=http://maven.apache.org/POM/4.0.0 \ + -u "/x:project/x:name" -v "$relativePath" \ + "$pom" > "${pom}.new" + mv "${pom}.new" "${pom}" + + # Add missing ones + xmlstarlet ed -P -N x=http://maven.apache.org/POM/4.0.0 \ + -s "/x:project[count(x:name)=0]" -t elem -n name -v "$relativePath" \ + "$pom" > "${pom}.new" + mv "${pom}.new" "${pom}" +} + +# Find all project poms ignoring the /src/ paths (We don't want to scan code) +find . -name pom.xml -not -path "*/src/*" | xargs -I^ -P8 bash -c "$(declare -f fix_name); fix_name ^" diff --git a/autorelease/scripts/fix-relativepaths.sh b/autorelease/scripts/fix-relativepaths.sh new file mode 100755 index 000000000..1c9b8d331 --- /dev/null +++ b/autorelease/scripts/fix-relativepaths.sh @@ -0,0 +1,62 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015, 2016 The Linux Foundation. All rights reserved. +############################################################################## + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +# MAP of path to a parent pom from the perspective of hosting directory +# starting from the autorelease repo root. +# +# Format: <groupId>:<artifactId>:<path> + +fix_relative_paths() { + PARENT_MAP=( + "org.openo.oparent:oparent:oparent" + "org.openo.nfvo:nfvo-root:nfvo" + ) + + pom=$1 + echo "Scanning $pom" + pomPath=`dirname $pom` + + # Find and replace parent poms + for parent in "${PARENT_MAP[@]}"; do + map=${parent#*:} # + + groupId=${parent%%:*} # Maven groupId + artifactId=${map%%:*} # Maven artifactId + projectPath=${map#*:} # Path to pom file from the perspective of hosting repo + + # Compute relative path to parent pom + relativePath=`python -c "import os.path; print os.path.relpath('$projectPath','$pomPath')"` + + # Standardize POM XML formatting + xmlstarlet fo "$pom" > "${pom}.old" + + # Update any existing relativePath values + xmlstarlet ed -P -N x=http://maven.apache.org/POM/4.0.0 \ + -u "//x:parent[x:artifactId=\"$artifactId\" and x:groupId=\"$groupId\"]/x:relativePath" \ + -v "$relativePath" "${pom}" > "${pom}.new1" + + # Add missing ones + xmlstarlet ed -P -N x=http://maven.apache.org/POM/4.0.0 \ + -s "//x:parent[x:artifactId=\"$artifactId\" and x:groupId=\"$groupId\" and count(x:relativePath)=0]" \ + -t elem -n relativePath -v "$relativePath" "${pom}.new1" > "${pom}.new2" + + # Standardize POM XML formatting again + xmlstarlet fo "${pom}.new2" > "${pom}.new" + + diff -u "${pom}.old" "${pom}.new" + cp "${pom}.new" "${pom}" + done +} + +# Find all project poms ignoring the /src/ paths (We don't want to scan code) +find . -name pom.xml -not -path "*/src/*" | sort | xargs -I^ bash -c "$(declare -f fix_relative_paths); fix_relative_paths ^" diff --git a/autorelease/scripts/generate-assembly.py b/autorelease/scripts/generate-assembly.py new file mode 100755 index 000000000..30d5e7516 --- /dev/null +++ b/autorelease/scripts/generate-assembly.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, csv, subprocess + +version = "1.1.0-SNAPSHOT" + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + + print """ +<!-- + Copyright (c) 2016-2017 Huawei Technologies Co., Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> + <id>linux64</id> + <formats> + <format>tar.gz</format> + </formats> + <fileSets> + <fileSet> + <directory>../../distribution</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>**</include> + </includes> + </fileSet> + </fileSets> + <dependencySets> +""" + + for row in reader: + if row["classifier"]: + include = "{}:{}:{}:{}".format(row["groupId"], row["artifactId"], row["extension"], row["classifier"]) + else: + include = "{}:{}:{}".format(row["groupId"], row["artifactId"], row["extension"]) + + txt = """ + <dependencySet> + <outputDirectory>{}</outputDirectory> + <useProjectArtifact>false</useProjectArtifact> + <includes> + <include>{}</include> + </includes> + <outputFileNameMapping>{}-${{artifact.version}}${{dashClassifier?}}.${{artifact.extension}}</outputFileNameMapping> + </dependencySet>""" + # print txt.format(row["filename"], include, row["filename"]) + + + print """ + </dependencySets> +</assembly> +""" diff --git a/autorelease/scripts/generate-binary-deps.py b/autorelease/scripts/generate-binary-deps.py new file mode 100755 index 000000000..aaf18001a --- /dev/null +++ b/autorelease/scripts/generate-binary-deps.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, csv, subprocess + +version = "1.1.0-SNAPSHOT" + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + errors = 0 + + items = [] + for row in reader: + txt = """ + <dependency> + <groupId>{}</groupId> + <artifactId>{}</artifactId> + <version>{}</version> + <type>{}</type> + <classifier>{}</classifier> + </dependency>""" + print txt.format(row["groupId"], row["artifactId"], version, row["extension"], row["classifier"]) + diff --git a/autorelease/scripts/generate-distribution-script.py b/autorelease/scripts/generate-distribution-script.py new file mode 100755 index 000000000..67cd26733 --- /dev/null +++ b/autorelease/scripts/generate-distribution-script.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, csv, subprocess + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() +url_template = "https://nexus.open-o.org/service/local/artifact/maven/redirect?r=releases&g={0}&a={1}&e={2}&c={3}&v=$VERSION" + +def parseRow(row): + service = row["service"] + filename = row["filename"] + groupId = row["groupId"] + artifactId = row["artifactId"] + extension = row["extension"] + classifier = row["classifier"] + url = url_template.format(groupId, artifactId, extension, classifier) + if classifier: + dest = "{}/{}-$VERSION.{}.{}".format(filename, filename, classifier, extension) + else: + dest = "{}/{}-$VERSION.{}".format(filename, filename, extension) + return {"url": url, "dest": dest} + + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + errors = 0 + + print """ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION=2.0.0 + +set +e + +""" + items = [] + for row in reader: + item = parseRow(row) + items.append(item) + + print "# {}".format(row["service"]) + print "mkdir -p {}".format(row["filename"]) + print "wget -O \"{}\" \"{}\"".format(item["dest"], item["url"]) + print "" diff --git a/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh new file mode 100755 index 000000000..9432cee54 --- /dev/null +++ b/autorelease/scripts/generate-jjbs/gen-code-jjbs.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build +JJB_DIR=$BUILD_DIR/ci-management/jjb + +cd $BUILD_DIR + +source $ROOT/scripts/generate-jjbs/workarounds.sh + + +find . -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do + project=${repo} + + toxs=`find $repo -type d -exec test -e "{}/tox.ini" ';' -prune -printf "%P/tox.ini\n" | sort` + + in_sun_branch=`git show sun:autorelease/all-projects.txt | grep -x ${repo}` + + mkdir -p $JJB_DIR/$repo + + if [ ! -z "$toxs" ]; then + rm -f $JJB_DIR/$repo/${repo}-python.yaml + + project=${repo}-java + echo $repo/${repo}-python.yaml + + cat > $JJB_DIR/$repo/${repo}-python.yaml <<EOF +--- +- project: + name: ${repo}-python + project: '${repo}' + stream: + - 'master': + branch: 'master' +EOF + if [ $in_sun_branch ]; then + cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF + - 'sun': + branch: 'sun' +EOF + fi + cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF + mvn-settings: '${repo}-settings' + build-node: 'centos7-redis-2c-1g' + subproject: +EOF + for tox in $toxs; do + toxpath=${tox%/tox.ini} + + if [ "$toxpath" == "" ]; then + subproject="root" + pathparam="." + pattern="**" + else + subproject=${toxpath////-} # replace slash with dash + pathparam=$toxpath + pattern="$toxpath/**" + + # do special subproject names + for SUB in "${SUBPROJECT_MAP[@]}"; do + if [ "${SUB%:*}" = "$repo/$toxpath" ]; then + subproject=${SUB#*:} + fi + done + fi + + cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF + - '${subproject}': + path: '${pathparam}' + pattern: '${pattern}' +EOF + done + cat >> $JJB_DIR/$repo/${repo}-python.yaml <<EOF + jobs: + - '{project}-{stream}-{subproject}-verify-python' +EOF + fi + + + + + has_subprojects=0 + for r in "${SPLIT_REPOS[@]}"; do + if [ "$repo" = "$r" ]; then + has_subprojects=1 + fi + done + + if [ $has_subprojects -eq 1 ]; then + poms=`find $repo -mindepth 1 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort` + else + poms=`find $repo -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort` + if [ "$poms" != "/pom.xml" ]; then + has_subprojects=1 + fi + fi + + + if [ ! -z "$poms" ]; then + rm -f $JJB_DIR/$repo/${repo}.yaml + rm -f $JJB_DIR/$repo/${repo}-java.yaml + echo $repo/${project}.yaml + fi + + if [ $has_subprojects -eq 0 ]; then + # root pom.xml found + cat > $JJB_DIR/$repo/${project}.yaml <<EOF +--- +- project: + name: ${project} + jobs: + - '{project}-{stream}-verify-java' + - '{project}-{stream}-merge-java' + + project: '${repo}' + stream: + - 'master': + branch: 'master' +EOF + if [ $in_sun_branch ]; then + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + - 'sun': + branch: 'sun' +EOF + fi + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + mvn-settings: '${repo}-settings' +EOF + elif [ ! -z "$poms" ]; then + cat > $JJB_DIR/$repo/${project}.yaml <<EOF +--- +- project: + name: ${project} + project: '${repo}' + stream: + - 'master': + branch: 'master' +EOF + if [ $in_sun_branch ]; then + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + - 'sun': + branch: 'sun' +EOF + fi + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + mvn-settings: '${repo}-settings' + subproject: +EOF + + for pom in $poms; do + pompath=${pom%/pom.xml} + subproject=${pompath////-} # replace slash with dash + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + - '${subproject}': + pom: '${pom}' + pattern: '${pompath}/**' +EOF + done + + if [ -e $BUILD_DIR/$repo/pom.xml ]; then + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + - 'root': + pom: 'pom.xml' + pattern: '*' +EOF + fi + + cat >> $JJB_DIR/$repo/${project}.yaml <<EOF + jobs: + - '{project}-{stream}-{subproject}-verify-java' + - '{project}-{stream}-{subproject}-merge-java' +EOF + fi +done diff --git a/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh new file mode 100755 index 000000000..539e91115 --- /dev/null +++ b/autorelease/scripts/generate-jjbs/gen-csit-jjbs.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# csit plans root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build +JJB_DIR=$BUILD_DIR/ci-management/jjb + +WORKSPACE=`git rev-parse --show-toplevel` +PLANS_DIR=`git rev-parse --show-toplevel`/test/csit/plans + +source $ROOT/scripts/generate-jjbs/workarounds.sh + + +TMPDIR=`mktemp -d --suffix="-docker-log"` +$ROOT/scripts/ls-microservice-repos.py | cut -d ' ' -f 1 > $TMPDIR/microservices.txt + +find $PLANS_DIR -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do + + OUTFILE=$JJB_DIR/$repo/${repo}-csit.yaml + cat > $OUTFILE <<EOF +--- +- project: + name: ${repo}-csit + jobs: + - 'integration-verify-{project}-csit-{functionality}' + - '{project}-csit-{functionality}' + project: '${repo}' + functionality: +EOF + find $PLANS_DIR/$repo -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read func; do + echo $repo / $func + cat >> $OUTFILE <<EOF + - '${func}': + trigger_jobs: +EOF + + docker kill `docker ps -a -q` + docker rm `docker ps -a -q` + + $WORKSPACE/test/csit/run-csit.sh plans/${repo}/${func} + mkdir -p $TMPDIR/${repo} + cp $WORKSPACE/archives/_docker-images.log $TMPDIR/${repo}/${func}.txt + + docker kill `docker ps -a -q` + docker rm `docker ps -a -q` + + + for image in `grep openoint $TMPDIR/${repo}/${func}.txt | grep -f $TMPDIR/microservices.txt | sort`; do + microservice=`echo $image | cut -d '/' -f 2` + cat >> $OUTFILE <<EOF + - 'integration-${microservice}-merge-docker' +EOF + done + + done + + cat >> $OUTFILE <<EOF + robot-options: '' + branch: 'master' +EOF +done + +echo $TMPDIR diff --git a/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh b/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh new file mode 100755 index 000000000..142892e72 --- /dev/null +++ b/autorelease/scripts/generate-jjbs/gen-docker-jjbs.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build +JJB_DIR=$BUILD_DIR/ci-management/jjb + +cd $BUILD_DIR + +# docker root dir +DOCKER=`git rev-parse --show-toplevel`/test/csit/docker + +cd $DOCKER + +OUTFILE=$JJB_DIR/integration/integration-docker-microservices.yaml + +cat > $OUTFILE <<EOF +--- +- project: + name: integration-docker-microservices + project: 'integration' + branch: 'master' + version: '1.1.0-SNAPSHOT' + mvn-settings: 'autorelease-settings' + build-node: 'centos7-robot-8c-8g' + jobs: + - 'integration-{microservice}-verify-docker' + - 'integration-{microservice}-merge-docker' + microservice: +EOF + +TMPDIR=`$ROOT/scripts/generate-jjbs/gen-job-lists.sh` +IFS=$'\n' +for line in `$ROOT/scripts/ls-microservice-repos.py | sort`; do + IFS=' ' + array=($line) + microservice=${array[0]} + repo=${array[1]} + trigger=${array[2]} + echo $microservice $repo $trigger + cat >> $OUTFILE <<EOF + - '${microservice}': + trigger_jobs: +EOF + for job in `cat ${TMPDIR}/merge-jobs/${repo}.txt | grep "${trigger}"`; do + cat >> $OUTFILE <<EOF + - '${job}' +EOF + done +done diff --git a/autorelease/scripts/generate-jjbs/gen-job-lists.sh b/autorelease/scripts/generate-jjbs/gen-job-lists.sh new file mode 100755 index 000000000..4e12c91dc --- /dev/null +++ b/autorelease/scripts/generate-jjbs/gen-job-lists.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build +JJB_DIR=$BUILD_DIR/ci-management/jjb + +cd $BUILD_DIR + +source $ROOT/scripts/generate-jjbs/workarounds.sh + + +TMPDIR=`mktemp -d` +echo $TMPDIR + +mkdir -p $TMPDIR/merge-jobs +find . -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | sort | while read repo; do + project=${repo} + OUTFILE=$TMPDIR/merge-jobs/${repo}.txt + + has_subprojects=0 + for r in "${SPLIT_REPOS[@]}"; do + if [ "$repo" = "$r" ]; then + has_subprojects=1 + fi + done + + if [ $has_subprojects -eq 1 ]; then + poms=`find $repo -mindepth 1 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort` + else + poms=`find $repo -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P/pom.xml\n" | sort` + if [ "$poms" != "/pom.xml" ]; then + has_subprojects=1 + fi + fi + + + if [ $has_subprojects -eq 0 ]; then + echo ${repo}-master-merge-java > $OUTFILE + elif [ ! -z "$poms" ]; then + for pom in $poms; do + pompath=${pom%/pom.xml} + subproject=${pompath////-} # replace slash with dash + echo ${repo}-master-${subproject}-merge-java >> $OUTFILE + done + fi +done + diff --git a/autorelease/scripts/generate-jjbs/workarounds.sh b/autorelease/scripts/generate-jjbs/workarounds.sh new file mode 100755 index 000000000..82b39658c --- /dev/null +++ b/autorelease/scripts/generate-jjbs/workarounds.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# specific workarounds to maintain existing job list +rm -f $BUILD_DIR/integration/autorelease/build/pom.xml +rm -f $BUILD_DIR/nfvo/monitor/pom.xml +touch $BUILD_DIR/common-tosca-aria/tox.ini + + +SUBPROJECT_MAP=( + "nfvo/drivers/vnfm/svnfm/zte/vmanager:drv-vnfm-zte" + "nfvo/drivers/vnfm/gvnfm/gvnfmadapter:drv-gvnfm" +) + +SPLIT_REPOS=("nfvo" "common-services-common-utilities" "multivimdriver-openstack" "multivimdriver-vmware-vio") diff --git a/autorelease/scripts/generate-pom.sh b/autorelease/scripts/generate-pom.sh new file mode 100755 index 000000000..5a3286a53 --- /dev/null +++ b/autorelease/scripts/generate-pom.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease +VERSION="1.1.0-SNAPSHOT" + +BUILD_DIR=$ROOT/build + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +FILE=$BUILD_DIR/pom-raw.xml + +cat > $FILE <<EOF +<!-- + Copyright (c) 2016-2017 Huawei Technologies Co., Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- AUTOGENERATED by generate-pom.sh; DO NOT MODIFY MANUALLY --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.openo.oparent</groupId> + <artifactId>oparent</artifactId> + <version>$VERSION</version> + <relativePath>oparent</relativePath> + </parent> + <groupId>org.openo.integration.distribution</groupId> + <artifactId>openo</artifactId> + <packaging>pom</packaging> + <modules> +EOF + +find -mindepth 1 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P\n" | sort | while read p; do + if [ $p != "integration/autorelease/build" ]; then + cat >> $FILE <<EOF + <module>$p</module> +EOF + fi +done + +cat >> $FILE <<EOF + </modules> + <dependencies> +EOF + +$ROOT/scripts/generate-binary-deps.py >> $FILE + +cat >> $FILE <<EOF + </dependencies> + + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.6</version> + <configuration> + <appendAssemblyId>true</appendAssemblyId> + <descriptors> + <descriptor>assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> +EOF + +xmlstarlet fo pom-raw.xml > pom.xml +rm -f pom-raw.xml + +$ROOT/scripts/generate-assembly.py > $BUILD_DIR/assembly.xml diff --git a/autorelease/scripts/get-all-repos.sh b/autorelease/scripts/get-all-repos.sh new file mode 100755 index 000000000..25a4e20e3 --- /dev/null +++ b/autorelease/scripts/get-all-repos.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright 2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +ssh -p 29418 gerrit.open-o.org gerrit ls-projects | grep -vf $ROOT/excluded-projects.txt diff --git a/autorelease/scripts/list-microservices.py b/autorelease/scripts/list-microservices.py new file mode 100755 index 000000000..03c0eb040 --- /dev/null +++ b/autorelease/scripts/list-microservices.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, csv, subprocess + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + + rows = [] + for row in reader: + rows.append(row) + + for row in rows: + print " * {:35s} {}".format(row["filename"], row["service"]) diff --git a/autorelease/scripts/ls-microservice-repos.py b/autorelease/scripts/ls-microservice-repos.py new file mode 100755 index 000000000..716bc3dc1 --- /dev/null +++ b/autorelease/scripts/ls-microservice-repos.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys, csv, subprocess, os, urllib2 + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + + for row in reader: + print "{} {} {}".format(row["filename"], row["repo"], row["job-trigger-string"]) diff --git a/autorelease/scripts/set-version.sh b/autorelease/scripts/set-version.sh new file mode 100755 index 000000000..3a4503a6d --- /dev/null +++ b/autorelease/scripts/set-version.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION="2.0.0" + +# autorelease root dir +ROOT=`git rev-parse --show-toplevel`/autorelease + +BUILD_DIR=$ROOT/build + +mkdir -p $BUILD_DIR +cd $BUILD_DIR + +if [ -z "$MVN" ]; then + export MVN=`which mvn` +fi + +# Jenkins sets a $MVN parameter via JJB that points to the appropriately installed maven +$MVN -q -f oparent/version/pom.xml versions:set versions:update-child-modules -DnewVersion=${VERSION} diff --git a/autorelease/scripts/verify-binaries.py b/autorelease/scripts/verify-binaries.py new file mode 100755 index 000000000..50f7312ef --- /dev/null +++ b/autorelease/scripts/verify-binaries.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# +# Copyright 2016-2017 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys, csv, subprocess + +root = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).rstrip() +path = "{}/autorelease/dist".format(root) +version = "1.1.0-SNAPSHOT" +url_template = "https://nexus.open-o.org/service/local/artifact/maven/redirect?r=snapshots&g={0}&a={1}&e={2}&c={3}&v=LATEST" + +subprocess.call(["rm", "-rf", path]) +subprocess.call(["mkdir", "-p", path]) + + +def parseRow(row): + service = row["service"] + filename = row["filename"] + groupId = row["groupId"] + artifactId = row["artifactId"] + extension = row["extension"] + classifier = row["classifier"] + url = url_template.format(groupId, artifactId, extension, classifier) + if classifier: + dest = "{}/{}-{}.{}.{}".format(path, filename, version, classifier, extension) + else: + dest = "{}/{}-{}.{}".format(path, filename, version, extension) + return {"url": url, "dest": dest} + + +with open( "{}/autorelease/binaries.csv".format(root), "r" ) as f: + reader = csv.DictReader(f) + errors = 0 + + items = [] + for row in reader: + item = parseRow(row) + items.append(item) + + result = subprocess.call(["wget", "-q", "--spider", "--content-disposition", item["url"]]) + if result == 0: + print "{} OK".format(row["service"]) + else: + errors += 1 + print "{} ERROR: {} not found".format(row["service"], item["url"]) + + print "{} errors found".format(errors) + + if errors > 0: + sys.exit(1) |