diff options
34 files changed, 1798 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..51ff612a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# Ignore Vim temporary files +.*.sw? + +# Ignore changes to the JJB Configuration File +jenkins.ini + +# We don't want .netrc files since they're credential files +.netrc + +# Maven +target/ + +# Vagrant +.vagrant + +# IntelliJ +.idea/ +*.iml diff --git a/.gitreview b/.gitreview new file mode 100644 index 000000000..9a76cf985 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.openecomp.org +port=29418 +project=ci-management.git diff --git a/jenkins-scripts/README b/jenkins-scripts/README new file mode 100644 index 000000000..4540d1dbd --- /dev/null +++ b/jenkins-scripts/README @@ -0,0 +1,8 @@ +The scripts in this directory are used by the Jenkins spin-up component +for dynamic minions. + +The spinup script will be as follows (${system_type} will be replaced +with the appropriate system_type script) + +git clone https://gerrit.open-o.org/r/p/ci-management.git /ci-management +/ci-management/jenkins-scripts/jenkins-init-script.sh diff --git a/jenkins-scripts/basic_settings.sh b/jenkins-scripts/basic_settings.sh new file mode 100755 index 000000000..25ccc6341 --- /dev/null +++ b/jenkins-scripts/basic_settings.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0> +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +case "$(facter operatingsystem)" in + Ubuntu) + apt-get update + ;; + *) + # Do nothing on other distros for now + ;; +esac + +IPADDR=$(facter ipaddress) +HOSTNAME=$(facter hostname) +FQDN=$(facter fqdn) + +echo "${IPADDR} ${HOSTNAME} ${FQDN}" >> /etc/hosts + +#Increase limits +cat <<EOF > /etc/security/limits.d/jenkins.conf +jenkins soft nofile 16000 +jenkins hard nofile 16000 +EOF + +cat <<EOSSH >> /etc/ssh/ssh_config +Host * + ServerAliveInterval 60 + +# we don't want to do SSH host key checking on spin-up systems +Host 10.30.104.* + StrictHostKeyChecking no + UserKnownHostsFile /dev/null +EOSSH + +cat <<EOKNOWN > /etc/ssh/ssh_known_hosts +[gerrit.openecomp.org]:29418 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCyAKv0UzEhpGKP/rW+yHpngl32Ppr5Uy42coz/sYZYxbtpI+9yaMqfoBb06ktmt6kV7OCT/Sc0OpyWmpcR0d7KZHxx/LE/nm7Gi+xkNHhb9G+Hn6DagP4V+LS6x1YlUt2InLCb8g07+/n6rfxqCI6emIJYu9aTpDhaARb+mMX1xzJuoa4wp59Yr1mkKK8lXHKGnPCemyl9a0vSRY58b7ZWG/N8giNvqYeptslIF1E/MEI5AP6nx7EupiVulAUdboAnDSD0urt9zdE8KRjboghB7PHguil6/OZhbqOb/uEt/rGCHn+02pig1K/vjFvCqNErNgS6EKj0IkH+cU/vjV6j +EOKNOWN + +# vim: sw=2 ts=2 sts=2 et : diff --git a/jenkins-scripts/create_jenkins_user.sh b/jenkins-scripts/create_jenkins_user.sh new file mode 100755 index 000000000..86a5c92d0 --- /dev/null +++ b/jenkins-scripts/create_jenkins_user.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0> +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +####################### +# Create Jenkins User # +####################### + +OS=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +useradd -m -s /bin/bash jenkins + +# Check if docker group exists +grep -q docker /etc/group +if [ "$?" == '0' ] +then + # Add jenkins user to docker group + usermod -a -G docker jenkins +fi + +# Check if mock group exists +grep -q mock /etc/group +if [ "$?" == '0' ] +then + # Add jenkins user to mock group so they can build Int/Pack's RPMs + usermod -a -G mock jenkins +fi + +mkdir /home/jenkins/.ssh +mkdir /w +cp -r /home/${OS}/.ssh/authorized_keys /home/jenkins/.ssh/authorized_keys +# Generate ssh key for use by Robot jobs +echo -e 'y\n' | ssh-keygen -N "" -f /home/jenkins/.ssh/id_rsa -t rsa +chown -R jenkins:jenkins /home/jenkins/.ssh /w diff --git a/jenkins-scripts/jenkins-init-script.sh b/jenkins-scripts/jenkins-init-script.sh new file mode 100755 index 000000000..67987b54d --- /dev/null +++ b/jenkins-scripts/jenkins-init-script.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0> +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# vim: ts=4 sw=4 sts=4 et : + +cd /ci-management/jenkins-scripts +chmod +x ./*.sh +./system_type.sh + +source /tmp/system_type.sh +./basic_settings.sh +if [ -f "${SYSTEM_TYPE}.sh" ] +then + ./"${SYSTEM_TYPE}.sh" +fi + +# Create the jenkins user last so that hopefully we don't have to deal with +# guard files +./create_jenkins_user.sh diff --git a/jenkins-scripts/system_type.sh b/jenkins-scripts/system_type.sh new file mode 100755 index 000000000..9aeea20d2 --- /dev/null +++ b/jenkins-scripts/system_type.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0> +############################################################################## +# Copyright (c) 2016 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +HOST=$(/bin/hostname) +SYSTEM_TYPE='' + +IFS=',' +for i in "basebuild,basebuild" \ + "centos,centos" \ + "trusty,trusty" \ + "xenial,xenial" +do set -- $i + if [[ $HOST == *"$1"* ]]; then + SYSTEM_TYPE="$2" + break + fi +done + +# Write out the system type to an environment file to then be sourced +echo "SYSTEM_TYPE=${SYSTEM_TYPE}" > /tmp/system_type.sh + +# vim: sw=4 ts=4 sts=4 et : diff --git a/jenkins.ini.example b/jenkins.ini.example new file mode 100644 index 000000000..c8486f89a --- /dev/null +++ b/jenkins.ini.example @@ -0,0 +1,12 @@ +[job_builder] +ignore_cache=True +keep_descriptions=False +include_path=.:scripts:~/git/ +recursive=True + +[jenkins] +#user=jenkins +#password=1234567890abcdef1234567890abcdef +url=http://localhost:8080 +##### This is deprecated, use job_builder section instead +#ignore_cache=True diff --git a/jjb/ci-management/ci-management-macros.yaml b/jjb/ci-management/ci-management-macros.yaml new file mode 100644 index 000000000..a5427ce06 --- /dev/null +++ b/jjb/ci-management/ci-management-macros.yaml @@ -0,0 +1,28 @@ +--- +# vim: sw=2 ts=2 sts=2 et : +- builder: + name: packer-validate + builders: + - config-file-provider: + files: + - file-id: 'packer-cloud-env' + variable: 'CLOUDENV' + - shell: | + cd packer + export PACKER_LOG="yes" + export PACKER_LOG_PATH="packer-validate.log" + packer.io validate -var-file=$CLOUDENV \ + -var-file=vars/{platform}.json \ + templates/{template}.json + +- builder: + name: packer-build + builders: + - shell: | + cd packer + export PACKER_LOG="yes" + export PACKER_LOG_PATH="packer-validate.log" + packer.io build -color=false \ + -var-file=$CLOUDENV \ + -var-file=vars/{platform}.json \ + templates/{template}.json diff --git a/jjb/ci-management/ci-management.yaml b/jjb/ci-management/ci-management.yaml new file mode 100644 index 000000000..d48fd490b --- /dev/null +++ b/jjb/ci-management/ci-management.yaml @@ -0,0 +1,206 @@ +--- +- project: + name: ci-management-jobs + project: 'ci-management' + jobs: + - '{project}-verify-jjb' + - '{project}-merge-jjb' + # - '{project}-verify-packer-{platforms}-{templates}' + # - '{project}-merge-packer-{platforms}-{templates}' + + archive-artifacts: '**/*.log' + branch: 'master' + build-timeout: '60' + + platforms: + - centos + - ubuntu-16.04 + + templates: + - basebuild + +- job-template: + name: '{project}-verify-jjb' + + project-type: freestyle + node: '{build-node}' + concurrent: true + + properties: + - ecomp-infra-properties: + build-days-to-keep: 14 + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: 'master' + refspec: 'refs/heads/master' + artifacts: '{archive-artifacts}' + + scm: + - gerrit-trigger-scm: + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-submitted: + server: '{server-name}' + project: '{project}' + branch: '{branch}' + files: 'jjb/**' + + builders: + - config-file-provider: + files: + - file-id: 'jjbini' + variable: 'JJBINI' + - shell: | + virtualenv $WORKSPACE/venv + source $WORKSPACE/venv/bin/activate + pip install --upgrade pip + pip freeze + pip install jenkins-job-builder + jenkins-jobs -l DEBUG --conf $JJBINI test -o archives/job_output jjb/ + gzip archives/job_output/* + - ci-management-check-unicode + +# publishers: +# - ecomp-infra-shiplogs: +# maven-version: 'mvn33' + +- job-template: + name: '{project}-merge-jjb' + + project-type: freestyle + node: '{build-node}' + + properties: + - ecomp-infra-properties: + build-days-to-keep: 14 + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: 'master' + refspec: 'refs/heads/master' + artifacts: '{archive-artifacts}' + + scm: + - gerrit-trigger-scm: + refspec: '' + choosing-strategy: 'default' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-merged: + server: '{server-name}' + project: '{project}' + branch: '{branch}' + files: 'jjb/**' + + builders: + - shell: | + virtualenv $WORKSPACE/venv + source $WORKSPACE/venv/bin/activate + pip install --upgrade pip + pip freeze + pip install jenkins-job-builder + jenkins-jobs --conf $JJBINI update --delete-old --workers 4 jjb/ + +# publishers: +# - ecomp-infra-shiplogs: +# maven-version: 'mvn33' + +- job-template: + name: '{project}-verify-packer-{platforms}-{templates}' + project-type: freestyle + node: '{build-node}' + concurrent: true + + properties: + - ecomp-infra-properties: + build-days-to-keep: 14 + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: 'master' + refspec: 'refs/heads/master' + artifacts: '{archive-artifacts}' + + scm: + - gerrit-trigger-scm: + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-submitted: + server: '{server-name}' + project: '{project}' + branch: 'master' + files: 'packer/**' + + builders: + - packer-validate: + platform: '{platforms}' + template: '{templates}' + +# publishers: +# - ecomp-infra-shiplogs: +# maven-version: 'mvn33' + +- job-template: + name: '{project}-merge-packer-{platforms}-{templates}' + project-type: freestyle + node: '{build-node}' + concurrent: true + + properties: + - ecomp-infra-properties: + build-days-to-keep: 14 + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: 'master' + refspec: 'refs/heads/master' + artifacts: '{archive-artifacts}' + + scm: + - gerrit-trigger-scm: + refspec: '' + choosing-strategy: 'default' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-merged: + server: '{server-name}' + project: '{project}' + branch: 'master' + files: 'packer/**' + + builders: + - packer-validate: + platform: '{platforms}' + template: '{templates}' + - packer-build: + platform: '{platforms}' + template: '{templates}' + +# publishers: +# - ecomp-infra-shiplogs: +# maven-version: 'mvn33' diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml new file mode 100644 index 000000000..99fc5a063 --- /dev/null +++ b/jjb/global-defaults.yaml @@ -0,0 +1,21 @@ +--- +# GLOBAL jenkins defaults + +- defaults: + name: global + + build-days-to-keep: 30 + # the below discarder values are hard coded into the macro and are only here + # for documentation purposes + build-num-to-keep: 40 + build-artifact-days-to-keep: -1 + build-artifact-num-to-keep: 5 + + # default gerrit server definition + server-name: 'ecomp' + + # Timeout in minutes + build-timeout: 360 + build-node: centos7-basebuild-2c-1g + + archive-artifacts: '' diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml new file mode 100644 index 000000000..d2068a828 --- /dev/null +++ b/jjb/global-macros.yaml @@ -0,0 +1,284 @@ +--- +# Global macros + +# BUILDERS +- builder: + name: ci-management-check-unicode + builders: + - shell: | + $WORKSPACE/scripts/check-unicode.sh jjb/ + +- builder: + name: provide-maven-settings + builders: + - config-file-provider: + files: + - file-id: '{global-settings-file}' + variable: 'GLOBAL_SETTINGS_FILE' + - file-id: '{settings-file}' + variable: 'SETTINGS_FILE' + +# PARAMETERS +- parameter: + name: ecomp-infra-parameters + parameters: + - string: + name: PROJECT + default: '{project}' + # yamllint disable rule:line-length + description: 'JJB configured PROJECT parameter to identify a Gerrit project' + # yamllint enable + - string: + name: ARCHIVE_ARTIFACTS + default: '{artifacts} **/target/surefire-reports/*-output.txt' + # yamllint disable rule:line-length + description: 'Space separated glob patterns for artifacts to archive to logs hosting' + # yamllint enable + - string: + name: GERRIT_BRANCH + default: '{branch}' + description: 'JJB configured GERRIT_BRANCH parameter' + - string: + name: GERRIT_PROJECT + default: '{project}' + description: 'GERRIT_PROJECT parameter if not given by trigger' + - string: + name: GERRIT_REFSPEC + default: '{refspec}' + description: 'GERRIT_REFSPEC parameter if not given by trigger' + - string: + name: STACK_NAME + default: '$JOB_NAME-$BUILD_NUMBER' + description: 'Used by Heat to generate a unique stack & vm name' + +- parameter: + name: maven-exec + parameters: + - string: + name: MVN + # yamllint disable rule:line-length + default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn' + # yamllint enable + description: 'Maven selector to be used by shell scripts' + +# PROPERTIES +- property: + name: ecomp-infra-properties + properties: + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: 40 + artifact-days-to-keep: -1 + artifact-num-to-keep: 5 + +# PUBLISHERS +- publisher: + name: archive-artifacts + publishers: + - archive: + artifacts: '{artifacts}' + allow-empty: true + fingerprint: true + latest-only: true + +- publisher: + name: email-notification + publishers: + - email-ext: + recipients: 'nobody@open-o.org' + reply-to: + content-type: default + # yamllint disable rule:line-length + subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!' + # yamllint enable + body: | + $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS: + + Check console output at $BUILD_URL to view the results. + unstable: true + fixed: true + send-to: + - developers + - recipients + +- publisher: + name: jacoco-report + publishers: + - jacoco: + exec-pattern: '**/**.exec' + class-pattern: '**/classes' + source-pattern: '**/src/main/java' + # yamllint disable rule:line-length + exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**' + # yamllint enable + status-update: true + targets: + - branch: + healthy: 10 + unhealthy: 20 + - method: + healthy: 50 + unhealthy: 40 + +- publisher: + name: ecomp-infra-shiplogs + # To archive things, the job will need to create an "archives" directory in + # the workspace and this macro will handle copying the contents of the + # archives directory. + # + # Uses the build parameter ARCHIVE_ARTIFACTS if not empty to find files to + # archive. You can pass globstar patterns for example "**/*.xml **/*.log" as + # the archive pattern. This is a space separated list of files to archive. + # + # Also ensure that the workspace is cleaned up at the end of the build. + publishers: + - postbuildscript: + builders: + - shell: !include-raw: include-raw-deploy-archives.sh + - maven-target: + maven-version: '{maven-version}' + pom: '.archives/deploy-archives.xml' + goals: 'clean deploy -V -B -q -Dmaven.repo.local=/tmp/r' + settings: 'jenkins-log-archives-settings' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + - description-setter: + regexp: '^Build logs: .*' + script-only-if-succeeded: false + script-only-if-failed: false + mark-unstable-if-failed: true + - workspace-cleanup: + fail-build: false` + +# SCMS +- scm: + name: git-scm + scm: + - git: + credentials-id: 'ecomp-jenkins-ssh' + url: '$GIT_BASE' + refspec: '' + branches: + - 'origin/${branch}' + skip-tag: true + wipe-workspace: true + timeout: 30 + +- scm: + name: gerrit-trigger-scm + scm: + - git: + credentials-id: 'ecomp-jenkins-ssh' + url: '$GIT_BASE' + refspec: '{refspec}' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + timeout: 30 + choosing-strategy: '{choosing-strategy}' + +- scm: + name: gerrit-trigger-scm-basedir + scm: + - git: + credentials-id: 'ecomp-jenkins-ssh' + url: '$GIT_BASE' + refspec: '{refspec}' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + timeout: 30 + choosing-strategy: '{choosing-strategy}' + basedir: '{basedir}' + +- scm: + name: git-branch-scm + scm: + - git: + credentials-id: 'ecomp-jenkins-ssh' + url: '$GIT_BASE' + refspec: '' + branches: + - 'origin/{branch}' + skip-tag: true + timeout: 30 + wipe-workspace: true + +# TRIGGERS + +# This is a single macro to use for all jobs who vote on every (relevant) patch +# set. Only 'recheck' trigger word is supported, it always triggers the full set +# of relevant jobs, in order to prevent Jenkins from starting only a subset and +# still voting Verified+1 +# Arguments: +# server: name of gerrit server to listen to +# project: pattern to match triggering projects +# branch: triggering branch name +# files: pattern to match triggering filepaths +- trigger: + name: gerrit-trigger-patch-submitted + triggers: + - gerrit: + server-name: '{server}' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '{files}' + +- trigger: + name: gerrit-trigger-patch-merged + triggers: + - gerrit: + server-name: '{server}' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '{files}' + +# WRAPPERS +- wrapper: + name: build-timeout + wrappers: + - timeout: + type: absolute + timeout: 360 + fail: true + +- wrapper: + # This wrapper is required for all jobs as it configures the wrappers + # needed by the eCOMP infra. + name: ecomp-infra-wrappers + wrappers: + - timeout: + type: absolute + timeout: '{build-timeout}' + timeout-var: 'BUILD_TIMEOUT' + fail: true + - ssh-agent-credentials: + users: + - 'ecomp-jenkins-ssh' + - openstack: + single-use: true diff --git a/jjb/global-templates-java.yaml b/jjb/global-templates-java.yaml new file mode 100644 index 000000000..4ca261f4e --- /dev/null +++ b/jjb/global-templates-java.yaml @@ -0,0 +1,243 @@ +--- +- job-template: + # Job template for Java verify jobs + # + # The purpose of this job template is to run "maven clean install" for + # projects using this template. + # + # Required Variables: + # branch: git branch (eg. stable/lithium or master) + + name: '{project}-{stream}-verify-java' + + project-type: freestyle + concurrent: true + node: '{build-node}' + + properties: + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: '{build-num-to-keep}' + artifact-days-to-keep: '{build-artifact-days-to-keep}' + artifact-num-to-keep: '{build-artifact-num-to-keep}' + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + + scm: + - gerrit-trigger-scm: + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-submitted: + name: '{project}' + branch: '{branch}' + + builders: + - provide-maven-settings: + global-settings-file: 'global-settings' + settings-file: '{mvn-settings}' + - maven-target: + maven-version: 'mvn33' + goals: 'clean install' + settings: '{mvn-settings}' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + + publishers: + - ecomp-infra-shiplogs: + maven-version: 'mvn33' + +- job-template: + # Job template for Java verify jobs with POM not at the root + # + # The purpose of this job template is to run "maven clean install" for + # projects using this template. + # + # Required Variables: + # branch: git branch (eg. stable/lithium or master) + # pom: name/location of the pom.xml file relative to the workspace + # pattern: ant file-path pattern relative to the workspace used to + # trigger the job + + name: '{project}-{stream}-{subproject}-verify-java' + + project-type: freestyle + concurrent: true + node: '{build-node}' + + properties: + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: '{build-num-to-keep}' + artifact-days-to-keep: '{build-artifact-days-to-keep}' + artifact-num-to-keep: '{build-artifact-num-to-keep}' + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + + scm: + - gerrit-trigger-scm: + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-submitted-pattern: + name: '{project}' + branch: '{branch}' + pattern: '{pattern}' + + builders: + - provide-maven-settings: + global-settings-file: 'global-settings' + settings-file: '{mvn-settings}' + - maven-target: + maven-version: 'mvn33' + pom: '{pom}' + goals: 'clean install' + settings: '{mvn-settings}' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + + publishers: + - ecomp-infra-shiplogs: + maven-version: 'mvn33' + +- job-template: + # Job template for Java merge jobs + # + # The purpose of this job template is to run "maven clean deploy" for + # projects using this template. + # + # Required Variables: + # branch: git branch (eg. stable/lithium or master) + name: '{project}-{stream}-merge-java' + + project-type: freestyle + node: '{build-node}' + + properties: + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: '{build-num-to-keep}' + artifact-days-to-keep: '{build-artifact-days-to-keep}' + artifact-num-to-keep: '{build-artifact-num-to-keep}' + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + + scm: + - gerrit-trigger-scm: + refspec: '' + choosing-strategy: 'default' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-merged: + name: '{project}' + branch: '{branch}' + + builders: + - provide-maven-settings: + global-settings-file: 'global-settings' + settings-file: '{mvn-settings}' + - maven-target: + maven-version: 'mvn33' + # yamllint disable rule:line-length + goals: 'clean deploy sonar:sonar -Dsonar.host.url=https://sonar.open-o.org' + # yamllint enable + settings: '{mvn-settings}' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + + publishers: + - ecomp-infra-shiplogs: + maven-version: 'mvn33' + +- job-template: + # Job template for Java merge jobs with POM not at the root + # + # The purpose of this job template is to run "maven clean deploy" for + # projects using this template. + # + # Required Variables: + # branch: git branch (eg. stable/lithium or master) + # pom: name/location of the pom.xml file relative to the workspace + # pattern: ant file-path pattern relative to the workspace used to + # trigger the job + + name: '{project}-{stream}-{subproject}-merge-java' + + project-type: freestyle + node: '{build-node}' + + properties: + - build-discarder: + days-to-keep: '{build-days-to-keep}' + num-to-keep: '{build-num-to-keep}' + artifact-days-to-keep: '{build-artifact-days-to-keep}' + artifact-num-to-keep: '{build-artifact-num-to-keep}' + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + + scm: + - gerrit-trigger-scm: + refspec: '' + choosing-strategy: 'default' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-merged-pattern: + name: '{project}' + branch: '{branch}' + pattern: '{pattern}' + + builders: + - provide-maven-settings: + global-settings-file: 'global-settings' + settings-file: '{mvn-settings}' + - maven-target: + maven-version: 'mvn33' + pom: '{pom}' + # yamllint disable rule:line-length + goals: 'clean deploy sonar:sonar -Dsonar.host.url=https://sonar.open-o.org' + # yamllint enable + settings: '{mvn-settings}' + settings-type: cfp + global-settings: 'global-settings' + global-settings-type: cfp + + publishers: + - ecomp-infra-shiplogs: + maven-version: 'mvn33' diff --git a/jjb/global-templates-python.yaml b/jjb/global-templates-python.yaml new file mode 100644 index 000000000..ef4cdeb76 --- /dev/null +++ b/jjb/global-templates-python.yaml @@ -0,0 +1,56 @@ +--- +- job-template: + # Job template for python verify jobs + # + # The purpose of this job template is to run tox for projects using this + # template. + # + # Required Variables: + # branch: git branch (eg. stable/lithium or master) + # Optional Variables: + # path: directory containing the project's tox.ini relative to + # the workspace. The default is the project root. + # pattern: ant file-path pattern relative to the workspace used to + # trigger the job + + name: '{project}-{stream}-{subproject}-verify-python' + path: '$WORKSPACE' + + project-type: freestyle + concurrent: true + node: '{build-node}' + + parameters: + - ecomp-infra-parameters: + project: '{project}' + branch: '{branch}' + refspec: 'refs/heads/{branch}' + + scm: + - gerrit-trigger-scm: + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ecomp-infra-wrappers: + build-timeout: '{build-timeout}' + + triggers: + - gerrit-trigger-patch-submitted-pattern: + name: '{project}' + branch: '{branch}' + pattern: '{pattern}' + + builders: + - shell: | + virtualenv $WORKSPACE/venv-tox + source $WORKSPACE/venv-tox/bin/activate + pip install --upgrade pip + pip install --upgrade tox argparse + pip freeze + cd $WORKSPACE/{path} + tox + + publishers: + - ecomp-infra-shiplogs: + maven-version: 'mvn33' diff --git a/jjb/include-raw-deploy-archives.sh b/jjb/include-raw-deploy-archives.sh new file mode 100644 index 000000000..bfe0bcbc7 --- /dev/null +++ b/jjb/include-raw-deploy-archives.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +set +e # Do not affect the build result if some part of archiving fails. + +ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" +[ "$LOGS_SERVER" ] || LOGS_SERVER="https://logs.open-o.org" +[ "$LOGS_REPO_URL" ] || LOGS_REPO_URL="https://nexus.open-o.org/service/local/repositories/logs" + +echo "Build logs: <a href=\"$LOGS_SERVER/$SILO/$ARCHIVES_DIR\">$LOGS_SERVER/$SILO/$ARCHIVES_DIR</a>" + +mkdir .archives +cd .archives/ + +cat > deploy-archives.xml <<EOF +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>logs</groupId> + <artifactId>logs</artifactId> + <version>1.0.0</version> + <packaging>pom</packaging> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8.2</version> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <plugin> + <groupId>org.sonatype.plugins</groupId> + <artifactId>maven-upload-plugin</artifactId> + <version>0.0.1</version> + <executions> + <execution> + <id>publish-site</id> + <phase>deploy</phase> + <goals> + <goal>upload-file</goal> + </goals> + <configuration> + <serverId>open-o-log-archives</serverId> + <repositoryUrl>$LOGS_REPO_URL/content-compressed</repositoryUrl> + <file>archives.zip</file> + <repositoryPath>$SILO</repositoryPath> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> +EOF + +mkdir -p $ARCHIVES_DIR +mkdir -p $WORKSPACE/archives +if [ ! -z "${{ARCHIVE_ARTIFACTS}}" ]; then + pushd $WORKSPACE + shopt -s globstar # Enable globstar to copy archives + archive_artifacts=$(echo ${{ARCHIVE_ARTIFACTS}}) + for f in $archive_artifacts; do + echo "Archiving $f" + mkdir -p $WORKSPACE/archives/$(dirname $f) + mv $f $WORKSPACE/archives/$f + done + shopt -u globstar # Disable globstar once archives are copied + popd +fi + + +# Ignore logging if archives doesn't exist +mv $WORKSPACE/archives/ $ARCHIVES_DIR > /dev/null 2>&1 +touch $ARCHIVES_DIR/_build-details.txt +echo "build-url: ${{BUILD_URL}}" >> $ARCHIVES_DIR/_build-details.txt +env > $ARCHIVES_DIR/_build-enviroment-variables.txt + +# capture system info +touch $ARCHIVES_DIR/_sys-info.txt +{{ + echo -e "uname -a:\n `uname -a` \n" + echo -e "df -h:\n `df -h` \n" + echo -e "free -m:\n `free -m` \n" + echo -e "nproc:\n `nproc` \n" + echo -e "lscpu:\n `lscpu` \n" + echo -e "ip addr:\n `/sbin/ip addr` \n" +}} 2>&1 | tee -a $ARCHIVES_DIR/_sys-info.txt + +# Magic string used to trim console logs at the appropriate level during wget +echo "-----END_OF_BUILD-----" +wget -O $ARCHIVES_DIR/console.log ${{BUILD_URL}}consoleText +wget -O $ARCHIVES_DIR/console-timestamp.log ${{BUILD_URL}}/timestamps?time=HH:mm:ss\&appendLog +sed -i '/^-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console.log +sed -i '/^.*-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console-timestamp.log + +gzip $ARCHIVES_DIR/*.txt $ARCHIVES_DIR/*.log +# find and gzip any 'text' files +find $ARCHIVES_DIR -type f -print0 \ + | xargs -0r file \ + | egrep -e ':.*text.*' \ + | cut -d: -f1 \ + | xargs -d'\n' -r gzip + +zip -r archives.zip $JENKINS_HOSTNAME/ +du -sh archives.zip diff --git a/packer/README.markdown b/packer/README.markdown new file mode 100644 index 000000000..4972affae --- /dev/null +++ b/packer/README.markdown @@ -0,0 +1,37 @@ +# Open-O Project CI Packer + +[Packer][1] is a tool for automatically creating VM and container images, +configuring them and post-processing them into standard output formats. + +We build Open-O's CI images via Packer. + +## Building + +You'll need to [install Packer][2], of course. + +Open-O's Packer configuration is divided into build-specific variables, +output-specific templates and a set of shared provisioning scripts. To do a +specific build, combine the template for the desired output artifact type with +a variable file. To build a new basebuild instance the following would be done: + +``` +packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/basebuild.json +``` + +**NOTE:** vars/cloud-env.json is a gitignored file as it contains private +information. There is a vars/cloud-env.json.example file that may be used as a +base for creating the one needed. + +This would build a bootable image in the Open-O CI cloud environment. + +From a high level, the builds: + +* Boot a specified base image in the cloud +* Run a set of shell scripts, listed in the template's shell provisioner + section, to do any configuration required by the builder. +* Execute a shutdown of the running instance +* Execute a 'nova image-create' operation against the shutdown instance. +* Perform a 'nova delete' operation against the shutdown instance. + +[1]: https://www.packer.io/ +[2]: https://www.packer.io/intro/getting-started/setup.html diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh new file mode 100644 index 000000000..68ba71cc6 --- /dev/null +++ b/packer/provision/basebuild.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + # Install python dependencies + yum install -y python-{devel,virtualenv,setuptools,pip} + + # Build dependencies for Python packages + yum install -y openssl-devel mysql-devel gcc + + # Autorelease support packages + yum install -y xmlstarlet + + # Additional libraries for Python ncclient + yum install -y libxml2 libxslt libxslt-devel libffi libffi-devel + + # Packer builds happen from the centos flavor images + PACKERDIR=$(mktemp -d) + # disable double quote checking + # shellcheck disable=SC2086 + cd $PACKERDIR + wget https://releases.hashicorp.com/packer/0.12.2/packer_0.12.2_linux_amd64.zip + unzip packer_0.12.2_linux_amd64.zip -d /usr/local/bin/ + # rename packer to avoid conflicts with cracklib + mv /usr/local/bin/packer /usr/local/bin/packer.io + + # cleanup from the installation + # disable double quote checking + # shellcheck disable=SC2086 + rm -rf $PACKERDIR + # cleanup from previous install process + if [ -d /tmp/packer ] + then + rm -rf /tmp/packer + fi +} + +ubuntu_systems() { + # Install python dependencies + apt-get install -y python-{dev,virtualenv,setuptools,pip} + + # Build dependencies for Python packages + apt-get install -y libssl-dev libmysqlclient-dev gcc + + # Additional libraries for Python ncclient + apt-get install -y wget unzip python-ncclient +} + +all_systems() { + echo 'No common distribution configuration to perform' +} + +echo "---> Detecting OS" +ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/basebuild/.dummy b/packer/provision/basebuild/.dummy new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/packer/provision/basebuild/.dummy diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh new file mode 100644 index 000000000..4870cbf74 --- /dev/null +++ b/packer/provision/baseline.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + # Handle the occurance where SELINUX is actually disabled + SELINUX=$(grep -E '^SELINUX=(disabled|permissive|enforcing)$' /etc/selinux/config) + MODE=$(echo "$SELINUX" | cut -f 2 -d '=') + case "$MODE" in + permissive) + echo "************************************" + echo "** SYSTEM ENTERING ENFORCING MODE **" + echo "************************************" + # make sure that the filesystem is properly labelled. + # it could be not fully labeled correctly if it was just switched + # from disabled, the autorelabel misses some things + # skip relabelling on /dev as it will generally throw errors + restorecon -R -e /dev / + + # enable enforcing mode from the very start + setenforce enforcing + + # configure system for enforcing mode on next boot + sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config + ;; + disabled) + sed -i 's/SELINUX=disabled/SELINUX=permissive/' /etc/selinux/config + touch /.autorelabel + + echo "*******************************************" + echo "** SYSTEM REQUIRES A RESTART FOR SELINUX **" + echo "*******************************************" + ;; + enforcing) + echo "*********************************" + echo "** SYSTEM IS IN ENFORCING MODE **" + echo "*********************************" + ;; + esac + + echo "---> Updating operating system" + yum clean all -q + yum install -y -q deltarpm + yum update -y -q + + # add in components we need or want on systems + echo "---> Installing base packages" + yum install -y -q @base https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + # separate group installs from package installs since a non-existing + # group with dnf based systems (F21+) will fail the install if such + # a group does not exist + yum install -y -q unzip xz puppet git git-review perl-XML-XPath wget make + + # All of our systems require Java (because of Jenkins) + # Install all versions of the OpenJDK devel but force 1.7.0 to be the + # default + + echo "---> Configuring OpenJDK" + yum install -y -q 'java-*-openjdk-devel' + + FACTER_OS=$(/usr/bin/facter operatingsystem) + FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) + case "$FACTER_OS" in + Fedora) + if [ "$FACTER_OSVER" -ge "21" ] + then + echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist" + else + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + fi + ;; + *) + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + ;; + esac + + # Needed to parse OpenStack commands used by infra stack commands + # to initialize Heat template based systems. + yum install -y jq +} + +ubuntu_systems() { + # Ignore SELinux since slamming that onto Ubuntu leads to + # frustration + + export DEBIAN_FRONTEND=noninteractive + cat <<EOF >> /etc/apt/apt.conf +APT { + Get { + Assume-Yes "true"; + allow-change-held-packages "true"; + allow-downgrades "true"; + allow-remove-essential "true"; + }; +}; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; + +EOF + + echo "---> Updating operating system" + apt-get update -qq > /dev/null + apt-get upgrade -qq > /dev/null + + # add in stuff we know we need + echo "---> Installing base packages" + apt-get install -qq unzip xz-utils puppet git git-review libxml-xpath-perl make wget > /dev/null + + # install Java 7 + echo "---> Configuring OpenJDK" + apt-get install -qq openjdk-7-jdk > /dev/null + + # make jdk8 available + add-apt-repository -y ppa:openjdk-r/ppa > /dev/null + apt-get update -qq > /dev/null + # We need to force openjdk-8-jdk to install + apt-get install -qq openjdk-8-jdk > /dev/null + + # make sure that we still default to openjdk 7 + update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java + update-alternatives --set javac /usr/lib/jvm/java-7-openjdk-amd64/bin/javac + + # Needed to parse OpenStack commands used by infra stack commands + # to initialize Heat template based systems. + apt-get install -qq jq > /dev/null + + # disable unattended upgrades & daily updates + echo '---> Disabling automatic daily upgrades' + sed -ine 's/"1"/"0"/g' /etc/apt/apt.conf.d/10periodic + echo 'APT::Periodic::Unattended-Upgrade "0";' >> /etc/apt/apt.conf.d/10periodic +} + +all_systems() { + # Allow jenkins access to update-alternatives command to switch java version + cat <<EOF >/etc/sudoers.d/89-jenkins-user-defaults +Defaults:jenkins !requiretty +jenkins ALL = NOPASSWD: /usr/bin/update-alternatives +EOF + + # Do any Distro specific installations here + echo "Checking distribution" + FACTER_OS=$(/usr/bin/facter operatingsystem) + case "$FACTER_OS" in + *) + echo "---> $FACTER_OS found" + echo "No extra steps for $FACTER_OS" + ;; + esac +} + +echo "---> Attempting to detect OS" +# upstream cloud images use the distro name as the initial user +ORIGIN=$(if [ -e /etc/redhat-release ] + then + echo redhat + else + echo ubuntu + fi) +#ORIGIN=$(logname) + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/null_data.sh b/packer/provision/null_data.sh new file mode 100644 index 000000000..3fa6a3c9f --- /dev/null +++ b/packer/provision/null_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +# Nothing to do for Ubuntu specific provisioning diff --git a/packer/provision/redis-server.sh b/packer/provision/redis-server.sh new file mode 100644 index 000000000..9a29b9547 --- /dev/null +++ b/packer/provision/redis-server.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + echo "---> Installing IUS repo and Redis" + # make sure that IUS is installed + yum install -y https://centos7.iuscommunity.org/ius-release.rpm + # now install redis 3.2.x + yum install -y redis32u + systemctl enable redis +} + +ubuntu_systems() { + echo "---> Installing Redis" + # Install redis-server + apt install -y redis-server +} + +all_systems() { + echo 'No common distribution configuration to perform' +} + +echo "---> Detecting OS" +ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/rh-user_data.sh b/packer/provision/rh-user_data.sh new file mode 100644 index 000000000..6bddb2449 --- /dev/null +++ b/packer/provision/rh-user_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +/bin/sed -i 's/ requiretty/ !requiretty/' /etc/sudoers; diff --git a/packer/provision/robot.sh b/packer/provision/robot.sh new file mode 100644 index 000000000..bac724a2e --- /dev/null +++ b/packer/provision/robot.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + # Assumes that python is already installed by basebuild + + # Install dependencies for robotframework and robotframework-sshlibrary + yum install -y -q yum-utils unzip sshuttle nc libffi-devel openssl-devel + + # Install docker + yum install -y docker supervisor bridge-utils + systemctl enable docker + + # configure docker networking so that it does not conflict with LF + # internal networks + cat <<EOL > /etc/sysconfig/docker-network +# /etc/sysconfig/docker-network +DOCKER_NETWORK_OPTIONS='--bip=10.250.0.254/24' +EOL + # configure docker daemon to listen on port 5555 enabling remote + # managment + sed -i -e "s#='--selinux-enabled'#='--selinux-enabled -H unix:///var/run/docker.sock -H tcp://0.0.0.0:5555'#g" /etc/sysconfig/docker + + # docker group doesn't get created by default for some reason + groupadd docker + + # Actual installation of robot is done from an integration JJB script +} + +ubuntu_systems() { + # Assumes that python is already installed by basebuild + + # Install dependencies for robotframework and robotframework-sshlibrary + apt install -y unzip sshuttle netcat libffi-dev libssl-dev + + # Install docker + apt install -y docker.io + + # Actual installation of robot is done from an integration JJB script +} + +all_systems() { + echo 'No common distribution configuration to perform' +} + +echo "---> Detecting OS" +ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/system_reseal.sh b/packer/provision/system_reseal.sh new file mode 100644 index 000000000..f8bc7dc5f --- /dev/null +++ b/packer/provision/system_reseal.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# vim: sw=2 ts=2 sts=2 et : + +rm -rf /etc/Pegasus/*.cnf /etc/Pegasus/*.crt /etc/Pegasus/*.csr \ + /etc/Pegasus/*.pem /etc/Pegasus/*.srl /root/anaconda-ks.cfg \ + /root/anaconda-post.log /root/initial-setup-ks.cfg /root/install.log \ + /root/install.log.syslog /var/cache/fontconfig/* /var/cache/gdm/* \ + /var/cache/man/* /var/lib/AccountService/users/* /var/lib/fprint/* \ + /var/lib/logrotate.status /var/log/*.log* /var/log/BackupPC/LOG \ + /var/log/ConsoleKit/* /var/log/anaconda.syslog /var/log/anaconda/* \ + /var/log/apache2/*_log /var/log/apache2/*_log-* /var/log/apt/* \ + /var/log/aptitude* /var/log/audit/* /var/log/btmp* /var/log/ceph/*.log \ + /var/log/chrony/*.log /var/log/cron* /var/log/cups/*_log /var/log/debug* \ + /var/log/dmesg* /var/log/exim4/* /var/log/faillog* /var/log/gdm/* \ + /var/log/glusterfs/*glusterd.vol.log /var/log/glusterfs/glusterfs.log \ + /var/log/httpd/*log /var/log/installer/* /var/log/jetty/jetty-console.log \ + /var/log/journal/* /var/log/lastlog* /var/log/libvirt/libvirtd.log \ + /var/log/libvirt/lxc/*.log /var/log/libvirt/qemu/*.log \ + /var/log/libvirt/uml/*.log /var/log/lightdm/* /var/log/mail/* \ + /var/log/maillog* /var/log/messages* /var/log/ntp /var/log/ntpstats/* \ + /var/log/ppp/connect-errors /var/log/rhsm/* /var/log/sa/* /var/log/secure* \ + /var/log/setroubleshoot/*.log /var/log/spooler* /var/log/squid/*.log \ + /var/log/syslog* /var/log/tallylog* /var/log/tuned/tuned.log /var/log/wtmp* \ + /var/named/data/named.run + +rm -rf ~/.viminfo /etc/ssh/ssh*key* /root/.ssh/* + +# kill any cloud-init related bits +rm -rf /var/lib/cloud/* + +# clean-up any manual packer uploads +rm -rf /tmp/packer + +# Force a system sync and sleep to get around any SSD issues +echo "Forcing sync and sleep for 10sec" +sync +sleep 10 diff --git a/packer/templates/basebuild.json b/packer/templates/basebuild.json new file mode 100644 index 000000000..426f290af --- /dev/null +++ b/packer/templates/basebuild.json @@ -0,0 +1,52 @@ +{ + "variables": { + "stack_tenant": null, + "stack_user": null, + "stack_pass": null, + "stack_network": null, + + "base_image": null, + "cloud_user": null, + "distro": null, + "cloud_user_data": null + }, + "builders": [ + { + "type": "openstack", + "identity_endpoint": "https://auth.vexxhost.net/v2.0/", + "tenant_name": "{{user `stack_tenant`}}", + "username": "{{user `stack_user`}}", + "password": "{{user `stack_pass`}}", + "region": "ca-ymq-1", + "ssh_username": "{{user `cloud_user`}}", + "image_name": "{{user `distro`}} - basebuild - {{isotime \"20060102-1504\"}}", + "source_image_name": "{{user `base_image`}}", + "flavor": "v1-standard-1", + "availability_zone": "ca-ymq-2", + "networks": [ + "{{user `stack_network`}}" + ], + "user_data_file": "{{user `cloud_user_data`}}" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /tmp/packer"] + }, + { + "type": "file", + "source": "provision/basebuild/", + "destination": "/tmp/packer" + }, + { + "type": "shell", + "scripts": [ + "provision/baseline.sh", + "provision/basebuild.sh", + "provision/system_reseal.sh" + ], + "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + } + ] +} diff --git a/packer/templates/redis.json b/packer/templates/redis.json new file mode 100644 index 000000000..df687ad08 --- /dev/null +++ b/packer/templates/redis.json @@ -0,0 +1,53 @@ +{ + "variables": { + "stack_tenant": null, + "stack_user": null, + "stack_pass": null, + "stack_network": null, + + "base_image": null, + "cloud_user": null, + "distro": null, + "cloud_user_data": null + }, + "builders": [ + { + "type": "openstack", + "identity_endpoint": "https://auth.vexxhost.net/v2.0/", + "tenant_name": "{{user `stack_tenant`}}", + "username": "{{user `stack_user`}}", + "password": "{{user `stack_pass`}}", + "region": "ca-ymq-1", + "ssh_username": "{{user `cloud_user`}}", + "image_name": "{{user `distro`}} - redis - {{isotime \"20060102-1504\"}}", + "source_image_name": "{{user `base_image`}}", + "flavor": "v1-standard-1", + "availability_zone": "ca-ymq-2", + "networks": [ + "{{user `stack_network`}}" + ], + "user_data_file": "{{user `cloud_user_data`}}" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /tmp/packer"] + }, + { + "type": "file", + "source": "provision/basebuild/", + "destination": "/tmp/packer" + }, + { + "type": "shell", + "scripts": [ + "provision/baseline.sh", + "provision/basebuild.sh", + "provision/redis-server.sh", + "provision/system_reseal.sh" + ], + "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + } + ] +} diff --git a/packer/templates/robot.json b/packer/templates/robot.json new file mode 100644 index 000000000..f31ad4618 --- /dev/null +++ b/packer/templates/robot.json @@ -0,0 +1,53 @@ +{ + "variables": { + "stack_tenant": null, + "stack_user": null, + "stack_pass": null, + "stack_network": null, + + "base_image": null, + "cloud_user": null, + "distro": null, + "cloud_user_data": null + }, + "builders": [ + { + "type": "openstack", + "identity_endpoint": "https://auth.vexxhost.net/v2.0/", + "tenant_name": "{{user `stack_tenant`}}", + "username": "{{user `stack_user`}}", + "password": "{{user `stack_pass`}}", + "region": "ca-ymq-1", + "ssh_username": "{{user `cloud_user`}}", + "image_name": "{{user `distro`}} - robot - {{isotime \"20060102-1504\"}}", + "source_image_name": "{{user `base_image`}}", + "flavor": "v1-standard-1", + "availability_zone": "ca-ymq-2", + "networks": [ + "{{user `stack_network`}}" + ], + "user_data_file": "{{user `cloud_user_data`}}" + } + ], + "provisioners": [ + { + "type": "shell", + "inline": ["mkdir -p /tmp/packer"] + }, + { + "type": "file", + "source": "provision/basebuild/", + "destination": "/tmp/packer" + }, + { + "type": "shell", + "scripts": [ + "provision/baseline.sh", + "provision/basebuild.sh", + "provision/robot.sh", + "provision/system_reseal.sh" + ], + "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" + } + ] +} diff --git a/packer/vars/.gitignore b/packer/vars/.gitignore new file mode 100644 index 000000000..af920284d --- /dev/null +++ b/packer/vars/.gitignore @@ -0,0 +1,2 @@ +# exclude private cloud-env settings +cloud-env.json diff --git a/packer/vars/centos.json b/packer/vars/centos.json new file mode 100644 index 000000000..7516bcc87 --- /dev/null +++ b/packer/vars/centos.json @@ -0,0 +1,7 @@ +{ + "base_image": "CentOS 7.1611 (2017-01-30) - LF upload", + "cloud_user": "centos", + + "distro": "CentOS 7", + "cloud_user_data": "provision/rh-user_data.sh" +} diff --git a/packer/vars/cloud-env.json.example b/packer/vars/cloud-env.json.example new file mode 100644 index 000000000..3a2ea9d2b --- /dev/null +++ b/packer/vars/cloud-env.json.example @@ -0,0 +1,6 @@ +{ + "stack_tenant": "TENANTID", + "stack_user": "USERID", + "stack_pass": "PASSWD", + "stack_network": "NETID", +} diff --git a/packer/vars/ubuntu-14.04.json b/packer/vars/ubuntu-14.04.json new file mode 100644 index 000000000..8acecd576 --- /dev/null +++ b/packer/vars/ubuntu-14.04.json @@ -0,0 +1,7 @@ +{ + "base_image": "Ubuntu 14.04 LTS (2016-07-22) - LF upload", + "cloud_user": "ubuntu", + + "distro": "Ubuntu 14.04", + "cloud_user_data": "provision/null_data.sh" +} diff --git a/packer/vars/ubuntu-16.04.json b/packer/vars/ubuntu-16.04.json new file mode 100644 index 000000000..c650d0737 --- /dev/null +++ b/packer/vars/ubuntu-16.04.json @@ -0,0 +1,7 @@ +{ + "base_image": "Ubuntu 16.04 LTS (2016-07-22) - LF upload", + "cloud_user": "ubuntu", + + "distro": "Ubuntu 16.04", + "cloud_user_data": "provision/null_data.sh" +} diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 000000000..c5ec4a4cb --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,7 @@ +# Python +__pycache__/ +*.pyc + +# ODL Signer +.sign-artifacts +gpg-signatures diff --git a/scripts/check-unicode.sh b/scripts/check-unicode.sh new file mode 100755 index 000000000..db45f6ac6 --- /dev/null +++ b/scripts/check-unicode.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# @License EPL-1.0 <http://spdx.org/licenses/EPL-1.0> +############################################################################## +# Copyright (c) 2015 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# Thanh Ha (The Linux Foundation) - Initial implementation +############################################################################## + +directory="." +if [ ! -z "$1" ]; then + directory="$1" +fi + +echo "Scanning $directory" +for x in $(find $directory -type f); do + if LC_ALL=C grep -q '[^[:print:][:space:]]' "$x"; then + echo "file "$x" contains non-ascii characters" + exit 1 + fi +done + +echo "All files are ASCII only" |