From 213a8351fd266294e17a862b4d223c1968800a09 Mon Sep 17 00:00:00 2001 From: Gary Wu Date: Fri, 16 Jun 2017 13:44:28 -0700 Subject: Added vagrant setup for a Jenkins system for ONAP Add a set of vagrant scripts that will set up a local Jenkins environment with pre-defined jobs to build all the ONAP java code and docker images. Change-Id: I67ed254bcb38a7be989b7c2861fd3ea4ec9c66f2 Signed-off-by: Gary Wu --- bootstrap/README.md | 6 +- bootstrap/jenkins/scripts/clone-all.sh | 5 + bootstrap/jenkins/scripts/gen-java-jobs.py | 27 ++++ bootstrap/jenkins/scripts/ls-projects.sh | 2 + bootstrap/jenkins/scripts/ls-top-poms.sh | 2 + bootstrap/jenkins/scripts/pull-all.sh | 8 + bootstrap/jenkins/vagrant/Vagrantfile | 75 +++++++++ bootstrap/jenkins/vagrant/bootstrap.sh | 43 +++++ bootstrap/jenkins/vagrant/build-all-docker.sh | 9 ++ bootstrap/jenkins/vagrant/build-all-java.sh | 9 ++ bootstrap/jenkins/vagrant/jenkins-init-1.sh | 26 +++ bootstrap/jenkins/vagrant/jenkins-init-2.sh | 15 ++ .../jenkins.install.InstallUtil.lastExecVersion | 1 + .../vagrant/jenkins/users/jenkins/config.xml | 38 +++++ bootstrap/jenkins/vagrant/jenkins_jobs.ini | 13 ++ bootstrap/jenkins/vagrant/jjb-init.sh | 34 ++++ bootstrap/jenkins/vagrant/jjb/docker-jobs.yaml | 76 +++++++++ .../jenkins/vagrant/jjb/include-docker-login.sh | 14 ++ bootstrap/jenkins/vagrant/jjb/java-jobs.yaml | 167 +++++++++++++++++++ bootstrap/jenkins/vagrant/jjb/template.yaml | 76 +++++++++ bootstrap/jenkins/vagrant/settings.xml | 177 +++++++++++++++++++++ 21 files changed, 822 insertions(+), 1 deletion(-) create mode 100755 bootstrap/jenkins/scripts/clone-all.sh create mode 100755 bootstrap/jenkins/scripts/gen-java-jobs.py create mode 100755 bootstrap/jenkins/scripts/ls-projects.sh create mode 100755 bootstrap/jenkins/scripts/ls-top-poms.sh create mode 100755 bootstrap/jenkins/scripts/pull-all.sh create mode 100644 bootstrap/jenkins/vagrant/Vagrantfile create mode 100755 bootstrap/jenkins/vagrant/bootstrap.sh create mode 100755 bootstrap/jenkins/vagrant/build-all-docker.sh create mode 100755 bootstrap/jenkins/vagrant/build-all-java.sh create mode 100755 bootstrap/jenkins/vagrant/jenkins-init-1.sh create mode 100755 bootstrap/jenkins/vagrant/jenkins-init-2.sh create mode 100644 bootstrap/jenkins/vagrant/jenkins/jenkins.install.InstallUtil.lastExecVersion create mode 100644 bootstrap/jenkins/vagrant/jenkins/users/jenkins/config.xml create mode 100644 bootstrap/jenkins/vagrant/jenkins_jobs.ini create mode 100755 bootstrap/jenkins/vagrant/jjb-init.sh create mode 100644 bootstrap/jenkins/vagrant/jjb/docker-jobs.yaml create mode 100644 bootstrap/jenkins/vagrant/jjb/include-docker-login.sh create mode 100644 bootstrap/jenkins/vagrant/jjb/java-jobs.yaml create mode 100644 bootstrap/jenkins/vagrant/jjb/template.yaml create mode 100644 bootstrap/jenkins/vagrant/settings.xml (limited to 'bootstrap') diff --git a/bootstrap/README.md b/bootstrap/README.md index 475aa9185..bad686157 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -3,5 +3,9 @@ ## Description -* A framework to automatically install and test a set of base infrastructure components for new developer +* A framework to automatically install and test a set of base infrastructure components and environments for new developers. + +## Sub-components + +* jenkins - A set of vagrant scripts that will set up a simple Jenkins environment with jobs configured to build all ONAP java code and docker images. diff --git a/bootstrap/jenkins/scripts/clone-all.sh b/bootstrap/jenkins/scripts/clone-all.sh new file mode 100755 index 000000000..6bdb68a09 --- /dev/null +++ b/bootstrap/jenkins/scripts/clone-all.sh @@ -0,0 +1,5 @@ +#!/bin/sh +while read p; do + echo $p + git clone ssh://gerrit.onap.org:29418/$p $p +done < projects.txt diff --git a/bootstrap/jenkins/scripts/gen-java-jobs.py b/bootstrap/jenkins/scripts/gen-java-jobs.py new file mode 100755 index 000000000..0da988083 --- /dev/null +++ b/bootstrap/jenkins/scripts/gen-java-jobs.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +import fileinput +import os +import subprocess + +print """- project: + name: onap-java + jobs: + - 'java-{project}' + project:""" + +for line in fileinput.input(): + repo = line.strip() + isGroupRepo = subprocess.call("grep -s 'Group repo' {}/README.md > /dev/null".format(repo), shell=True) == 0 + if not isGroupRepo: + pompaths = os.popen("./ls-top-poms.sh {}".format(repo)).readlines() + for pompath in pompaths: + pompath = pompath.strip() + project = repo.replace("/", "_") + if pompath: + project += "_" + pompath.replace("/", "_") + print " - '{}':".format(project) + print " repo: '{}'".format(repo) + if pompath: + print " pom: '{}/pom.xml'".format(pompath) + else: + print " pom: 'pom.xml'" diff --git a/bootstrap/jenkins/scripts/ls-projects.sh b/bootstrap/jenkins/scripts/ls-projects.sh new file mode 100755 index 000000000..17b8c7893 --- /dev/null +++ b/bootstrap/jenkins/scripts/ls-projects.sh @@ -0,0 +1,2 @@ +#!/bin/sh +ssh -p 29418 gerrit.onap.org gerrit ls-projects | grep -v All diff --git a/bootstrap/jenkins/scripts/ls-top-poms.sh b/bootstrap/jenkins/scripts/ls-top-poms.sh new file mode 100755 index 000000000..e856ee977 --- /dev/null +++ b/bootstrap/jenkins/scripts/ls-top-poms.sh @@ -0,0 +1,2 @@ +#!/bin/sh +find $1 -mindepth 0 -type d -exec test -e "{}/pom.xml" ';' -prune -printf "%P\n" | sort diff --git a/bootstrap/jenkins/scripts/pull-all.sh b/bootstrap/jenkins/scripts/pull-all.sh new file mode 100755 index 000000000..cf6655ce7 --- /dev/null +++ b/bootstrap/jenkins/scripts/pull-all.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while read p; do + echo $p + cd ~/Projects/onap/$p + git fetch + git reset --hard origin + git clean -f -d -x +done < projects.txt diff --git a/bootstrap/jenkins/vagrant/Vagrantfile b/bootstrap/jenkins/vagrant/Vagrantfile new file mode 100644 index 000000000..02b3ec18c --- /dev/null +++ b/bootstrap/jenkins/vagrant/Vagrantfile @@ -0,0 +1,75 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure(2) do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "ubuntu/xenial64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 8080, host: 8080 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # # Display the VirtualBox GUI when booting the machine + # vb.gui = true + # + # # Customize the amount of memory on the VM: + vb.memory = "8192" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + # config.vm.provision "shell", inline: <<-SHELL + # sudo apt-get update + # sudo apt-get install -y apache2 + # SHELL + config.vm.provision :shell, path: "bootstrap.sh" + + # requires vagrant plugin install vagrant-disksize + config.disksize.size = '80GB' +end diff --git a/bootstrap/jenkins/vagrant/bootstrap.sh b/bootstrap/jenkins/vagrant/bootstrap.sh new file mode 100755 index 000000000..ab5c45f32 --- /dev/null +++ b/bootstrap/jenkins/vagrant/bootstrap.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +sed -i 's|archive\.ubuntu\.com|mirrors.ocf.berkeley.edu|g' /etc/apt/sources.list + +echo "192.168.33.1 nexus-proxy" >> /etc/hosts + + +cat >> /etc/inputrc < /etc/apt/sources.list.d/jenkins.list' +apt-get update +apt-get -y install jenkins jenkins-job-builder python-pip + +apt-get -y install docker.io +sudo usermod -aG docker ubuntu +sudo usermod -aG docker jenkins + +su -l jenkins -c "/vagrant/jenkins-init-1.sh" + +sudo systemctl restart jenkins +sleep 5 + +su -l jenkins -c "/vagrant/jenkins-init-2.sh" + +sudo systemctl restart jenkins +sleep 5 + +su -l jenkins -c "/vagrant/jjb-init.sh" + diff --git a/bootstrap/jenkins/vagrant/build-all-docker.sh b/bootstrap/jenkins/vagrant/build-all-docker.sh new file mode 100755 index 000000000..081bb1246 --- /dev/null +++ b/bootstrap/jenkins/vagrant/build-all-docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# build all jobs +cd ~jenkins +for d in jobs/docker*; do + JOB=$(basename "$d") + echo build "$JOB" + java -jar jenkins-cli.jar -s http://localhost:8080/ -auth jenkins:jenkins build "$JOB" +done diff --git a/bootstrap/jenkins/vagrant/build-all-java.sh b/bootstrap/jenkins/vagrant/build-all-java.sh new file mode 100755 index 000000000..67333b76f --- /dev/null +++ b/bootstrap/jenkins/vagrant/build-all-java.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# build all jobs +cd ~jenkins +for d in jobs/java*; do + JOB=$(basename "$d") + echo build "$JOB" + java -jar jenkins-cli.jar -s http://localhost:8080/ -auth jenkins:jenkins build "$JOB" +done diff --git a/bootstrap/jenkins/vagrant/jenkins-init-1.sh b/bootstrap/jenkins/vagrant/jenkins-init-1.sh new file mode 100755 index 000000000..a5599a7e8 --- /dev/null +++ b/bootstrap/jenkins/vagrant/jenkins-init-1.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +git config --global user.email "jenkins@localhost" +git config --global user.name "jenkins" + +cd ~/jenkins + +cp /etc/skel/.profile . +cat > .bashrc < + + jenkins + + + {AQAAABAAAAAwQAGpldGajxw//dhxd53gZGv4w0JnZYDETTLBQdpotQXt02s0lq13YrhyaytbLFMflb98hzWY9YBlDIThZt7u+Q==} + + + + + + + + + all + false + false + + + + + + default + + + + + + false + + + #jbcrypt:$2a$10$Esc9z/mnK/CQ8crgFbE3/eP1EI6pvzIHRBe3SSik7rrNt.DDftON2 + + + jenkins@localhost + + + diff --git a/bootstrap/jenkins/vagrant/jenkins_jobs.ini b/bootstrap/jenkins/vagrant/jenkins_jobs.ini new file mode 100644 index 000000000..5cc6dc9f3 --- /dev/null +++ b/bootstrap/jenkins/vagrant/jenkins_jobs.ini @@ -0,0 +1,13 @@ +[job_builder] +ignore_cache=True +keep_descriptions=False +include_path=.:scripts:~/git/ +recursive=False +exclude=.*:manual:./development +allow_duplicates=False + +[jenkins] +user=jenkins +password=jenkins +url=http://localhost:8080 +query_plugins_info=False diff --git a/bootstrap/jenkins/vagrant/jjb-init.sh b/bootstrap/jenkins/vagrant/jjb-init.sh new file mode 100755 index 000000000..01cfcfada --- /dev/null +++ b/bootstrap/jenkins/vagrant/jjb-init.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +git add -A +git commit -m 'Installed plugins, restarted Jenkins' + + +mkdir -p ~/.config/jenkins_jobs +cp /vagrant/jenkins_jobs.ini ~/.config/jenkins_jobs + +pip install --user jenkins-job-builder + +jenkins-job-builder update -r /vagrant/jjb + +cat > .gitignore < + + + + + + onap-snapshots + + + onap-snapshots + onap-snapshots + https://nexus.onap.org/content/repositories/snapshots/ + + false + + + true + + + + + + onap-snapshots + onap-snapshots + https://nexus.onap.org/content/repositories/snapshots/ + + false + + + true + + + + + + onap-staging + + + onap-staging + onap-staging + https://nexus.onap.org/content/repositories/staging/ + + true + + + false + + + + + + onap-staging + onap-staging + https://nexus.onap.org/content/repositories/staging/ + + true + + + false + + + + + + onap-releases + + + onap-releases + onap-releases + https://nexus.onap.org/content/repositories/releases/ + + true + + + false + + + + + + onap-releases + onap-releases + https://nexus.onap.org/content/repositories/releases/ + + true + + + false + + + + + + onap-public + + + onap-public + onap-public + https://nexus.onap.org/content/repositories/public/ + + true + + + false + + + + + + onap-public + onap-public + https://nexus.onap.org/content/repositories/public/ + + true + + + false + + + + + + local-public + + + local-public + local-public + http://nexus-proxy:8081/nexus/content/repositories/public/ + + true + + + false + + + + + + local-public + local-public + http://nexus-proxy:8081/nexus/content/repositories/public/ + + true + + + false + + + + + + + + onap-snapshots + onap-staging + onap-releases + + local-public + + + -- cgit 1.2.3-korg