Initial ci-management upload

* Configure initial jobs and validate Jenkins environment
* Do not hook up packer build jobs at this time

Change-Id: I1818e8680d215318410f6beff5af054db03e7fa1
Signed-off-by: Andrew Grimberg <agrimberg@linuxfoundation.org>

7 files changed, 944 insertions, 0 deletions

diff --git a/jjb/ci-management/ci-management-macros.yaml b/jjb/ci-management/ci-management-macros.yaml
new file mode 100644
index 000000000..a5427ce06
--- /dev/null
+++ b/jjb/ci-management/ci-management-macros.yaml
@@ -0,0 +1,28 @@
+---
+# vim: sw=2 ts=2 sts=2 et :
+- builder:
+    name: packer-validate
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: 'packer-cloud-env'
+              variable: 'CLOUDENV'
+      - shell: |
+          cd packer
+          export PACKER_LOG="yes"
+          export PACKER_LOG_PATH="packer-validate.log"
+          packer.io validate -var-file=$CLOUDENV \
+            -var-file=vars/{platform}.json \
+            templates/{template}.json
+
+- builder:
+    name: packer-build
+    builders:
+      - shell: |
+          cd packer
+          export PACKER_LOG="yes"
+          export PACKER_LOG_PATH="packer-validate.log"
+          packer.io build -color=false \
+            -var-file=$CLOUDENV \
+            -var-file=vars/{platform}.json \
+            templates/{template}.json
diff --git a/jjb/ci-management/ci-management.yaml b/jjb/ci-management/ci-management.yaml
new file mode 100644
index 000000000..d48fd490b
--- /dev/null
+++ b/jjb/ci-management/ci-management.yaml
@@ -0,0 +1,206 @@
+---
+- project:
+    name: ci-management-jobs
+    project: 'ci-management'
+    jobs:
+      - '{project}-verify-jjb'
+      - '{project}-merge-jjb'
+      # - '{project}-verify-packer-{platforms}-{templates}'
+      # - '{project}-merge-packer-{platforms}-{templates}'
+
+    archive-artifacts: '**/*.log'
+    branch: 'master'
+    build-timeout: '60'
+
+    platforms:
+      - centos
+      - ubuntu-16.04
+
+    templates:
+      - basebuild
+
+- job-template:
+    name: '{project}-verify-jjb'
+
+    project-type: freestyle
+    node: '{build-node}'
+    concurrent: true
+
+    properties:
+      - ecomp-infra-properties:
+          build-days-to-keep: 14
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: 'master'
+          refspec: 'refs/heads/master'
+          artifacts: '{archive-artifacts}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-submitted:
+          server: '{server-name}'
+          project: '{project}'
+          branch: '{branch}'
+          files: 'jjb/**'
+
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: 'jjbini'
+              variable: 'JJBINI'
+      - shell: |
+          virtualenv $WORKSPACE/venv
+          source $WORKSPACE/venv/bin/activate
+          pip install --upgrade pip
+          pip freeze
+          pip install jenkins-job-builder
+          jenkins-jobs -l DEBUG --conf $JJBINI test -o archives/job_output jjb/
+          gzip archives/job_output/*
+      - ci-management-check-unicode
+
+#     publishers:
+#       - ecomp-infra-shiplogs:
+#           maven-version: 'mvn33'
+
+- job-template:
+    name: '{project}-merge-jjb'
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    properties:
+      - ecomp-infra-properties:
+          build-days-to-keep: 14
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: 'master'
+          refspec: 'refs/heads/master'
+          artifacts: '{archive-artifacts}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: ''
+          choosing-strategy: 'default'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-merged:
+          server: '{server-name}'
+          project: '{project}'
+          branch: '{branch}'
+          files: 'jjb/**'
+
+    builders:
+      - shell: |
+          virtualenv $WORKSPACE/venv
+          source $WORKSPACE/venv/bin/activate
+          pip install --upgrade pip
+          pip freeze
+          pip install jenkins-job-builder
+          jenkins-jobs --conf $JJBINI update --delete-old --workers 4 jjb/
+
+#     publishers:
+#       - ecomp-infra-shiplogs:
+#           maven-version: 'mvn33'
+
+- job-template:
+    name: '{project}-verify-packer-{platforms}-{templates}'
+    project-type: freestyle
+    node: '{build-node}'
+    concurrent: true
+
+    properties:
+      - ecomp-infra-properties:
+          build-days-to-keep: 14
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: 'master'
+          refspec: 'refs/heads/master'
+          artifacts: '{archive-artifacts}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-submitted:
+          server: '{server-name}'
+          project: '{project}'
+          branch: 'master'
+          files: 'packer/**'
+
+    builders:
+      - packer-validate:
+          platform: '{platforms}'
+          template: '{templates}'
+
+#     publishers:
+#       - ecomp-infra-shiplogs:
+#           maven-version: 'mvn33'
+
+- job-template:
+    name: '{project}-merge-packer-{platforms}-{templates}'
+    project-type: freestyle
+    node: '{build-node}'
+    concurrent: true
+
+    properties:
+      - ecomp-infra-properties:
+          build-days-to-keep: 14
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: 'master'
+          refspec: 'refs/heads/master'
+          artifacts: '{archive-artifacts}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: ''
+          choosing-strategy: 'default'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-merged:
+          server: '{server-name}'
+          project: '{project}'
+          branch: 'master'
+          files: 'packer/**'
+
+    builders:
+      - packer-validate:
+          platform: '{platforms}'
+          template: '{templates}'
+      - packer-build:
+          platform: '{platforms}'
+          template: '{templates}'
+
+#     publishers:
+#       - ecomp-infra-shiplogs:
+#           maven-version: 'mvn33'
diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml
new file mode 100644
index 000000000..99fc5a063
--- /dev/null
+++ b/jjb/global-defaults.yaml
@@ -0,0 +1,21 @@
+---
+# GLOBAL jenkins defaults
+
+- defaults:
+    name: global
+
+    build-days-to-keep: 30
+    # the below discarder values are hard coded into the macro and are only here
+    # for documentation purposes
+    build-num-to-keep: 40
+    build-artifact-days-to-keep: -1
+    build-artifact-num-to-keep: 5
+
+    # default gerrit server definition
+    server-name: 'ecomp'
+
+    # Timeout in minutes
+    build-timeout: 360
+    build-node: centos7-basebuild-2c-1g
+
+    archive-artifacts: ''
diff --git a/jjb/global-macros.yaml b/jjb/global-macros.yaml
new file mode 100644
index 000000000..d2068a828
--- /dev/null
+++ b/jjb/global-macros.yaml
@@ -0,0 +1,284 @@
+---
+# Global macros
+
+# BUILDERS
+- builder:
+    name: ci-management-check-unicode
+    builders:
+      - shell: |
+          $WORKSPACE/scripts/check-unicode.sh jjb/
+
+- builder:
+    name: provide-maven-settings
+    builders:
+      - config-file-provider:
+          files:
+            - file-id: '{global-settings-file}'
+              variable: 'GLOBAL_SETTINGS_FILE'
+            - file-id: '{settings-file}'
+              variable: 'SETTINGS_FILE'
+
+# PARAMETERS
+- parameter:
+    name: ecomp-infra-parameters
+    parameters:
+      - string:
+          name: PROJECT
+          default: '{project}'
+          # yamllint disable rule:line-length
+          description: 'JJB configured PROJECT parameter to identify a Gerrit project'
+          # yamllint enable
+      - string:
+          name: ARCHIVE_ARTIFACTS
+          default: '{artifacts} **/target/surefire-reports/*-output.txt'
+          # yamllint disable rule:line-length
+          description: 'Space separated glob patterns for artifacts to archive to logs hosting'
+          # yamllint enable
+      - string:
+          name: GERRIT_BRANCH
+          default: '{branch}'
+          description: 'JJB configured GERRIT_BRANCH parameter'
+      - string:
+          name: GERRIT_PROJECT
+          default: '{project}'
+          description: 'GERRIT_PROJECT parameter if not given by trigger'
+      - string:
+          name: GERRIT_REFSPEC
+          default: '{refspec}'
+          description: 'GERRIT_REFSPEC parameter if not given by trigger'
+      - string:
+          name: STACK_NAME
+          default: '$JOB_NAME-$BUILD_NUMBER'
+          description: 'Used by Heat to generate a unique stack & vm name'
+
+- parameter:
+    name: maven-exec
+    parameters:
+      - string:
+          name: MVN
+          # yamllint disable rule:line-length
+          default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
+          # yamllint enable
+          description: 'Maven selector to be used by shell scripts'
+
+# PROPERTIES
+- property:
+    name: ecomp-infra-properties
+    properties:
+      - build-discarder:
+          days-to-keep: '{build-days-to-keep}'
+          num-to-keep: 40
+          artifact-days-to-keep: -1
+          artifact-num-to-keep: 5
+
+# PUBLISHERS
+- publisher:
+    name: archive-artifacts
+    publishers:
+      - archive:
+          artifacts: '{artifacts}'
+          allow-empty: true
+          fingerprint: true
+          latest-only: true
+
+- publisher:
+    name: email-notification
+    publishers:
+      - email-ext:
+          recipients: 'nobody@open-o.org'
+          reply-to:
+          content-type: default
+          # yamllint disable rule:line-length
+          subject: '{email-prefix} $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS!'
+          # yamllint enable
+          body: |
+            $PROJECT_NAME - Build # $BUILD_NUMBER - $BUILD_STATUS:
+
+            Check console output at $BUILD_URL to view the results.
+          unstable: true
+          fixed: true
+          send-to:
+            - developers
+            - recipients
+
+- publisher:
+    name: jacoco-report
+    publishers:
+      - jacoco:
+          exec-pattern: '**/**.exec'
+          class-pattern: '**/classes'
+          source-pattern: '**/src/main/java'
+          # yamllint disable rule:line-length
+          exclusion-pattern: '**/gen**,**/generated-sources/**,**/yang-gen**,**/pax/**'
+          # yamllint enable
+          status-update: true
+          targets:
+            - branch:
+                healthy: 10
+                unhealthy: 20
+            - method:
+                healthy: 50
+                unhealthy: 40
+
+- publisher:
+    name: ecomp-infra-shiplogs
+    # To archive things, the job will need to create an "archives" directory in
+    # the workspace and this macro will handle copying the contents of the
+    # archives directory.
+    #
+    # Uses the build parameter ARCHIVE_ARTIFACTS if not empty to find files to
+    # archive. You can pass globstar patterns for example "**/*.xml **/*.log" as
+    # the archive pattern. This is a space separated list of files to archive.
+    #
+    # Also ensure that the workspace is cleaned up at the end of the build.
+    publishers:
+      - postbuildscript:
+          builders:
+            - shell: !include-raw: include-raw-deploy-archives.sh
+            - maven-target:
+                maven-version: '{maven-version}'
+                pom: '.archives/deploy-archives.xml'
+                goals: 'clean deploy -V -B -q -Dmaven.repo.local=/tmp/r'
+                settings: 'jenkins-log-archives-settings'
+                settings-type: cfp
+                global-settings: 'global-settings'
+                global-settings-type: cfp
+            - description-setter:
+                regexp: '^Build logs: .*'
+          script-only-if-succeeded: false
+          script-only-if-failed: false
+          mark-unstable-if-failed: true
+      - workspace-cleanup:
+          fail-build: false`
+
+# SCMS
+- scm:
+    name: git-scm
+    scm:
+      - git:
+          credentials-id: 'ecomp-jenkins-ssh'
+          url: '$GIT_BASE'
+          refspec: ''
+          branches:
+            - 'origin/${branch}'
+          skip-tag: true
+          wipe-workspace: true
+          timeout: 30
+
+- scm:
+    name: gerrit-trigger-scm
+    scm:
+      - git:
+          credentials-id: 'ecomp-jenkins-ssh'
+          url: '$GIT_BASE'
+          refspec: '{refspec}'
+          branches:
+            - 'origin/$GERRIT_BRANCH'
+          skip-tag: true
+          timeout: 30
+          choosing-strategy: '{choosing-strategy}'
+
+- scm:
+    name: gerrit-trigger-scm-basedir
+    scm:
+      - git:
+          credentials-id: 'ecomp-jenkins-ssh'
+          url: '$GIT_BASE'
+          refspec: '{refspec}'
+          branches:
+            - 'origin/$GERRIT_BRANCH'
+          skip-tag: true
+          timeout: 30
+          choosing-strategy: '{choosing-strategy}'
+          basedir: '{basedir}'
+
+- scm:
+    name: git-branch-scm
+    scm:
+      - git:
+          credentials-id: 'ecomp-jenkins-ssh'
+          url: '$GIT_BASE'
+          refspec: ''
+          branches:
+            - 'origin/{branch}'
+          skip-tag: true
+          timeout: 30
+          wipe-workspace: true
+
+# TRIGGERS
+
+# This is a single macro to use for all jobs who vote on every (relevant) patch
+# set. Only 'recheck' trigger word is supported, it always triggers the full set
+# of relevant jobs, in order to prevent Jenkins from starting only a subset and
+# still voting Verified+1
+# Arguments:
+#     server: name of gerrit server to listen to
+#     project: pattern to match triggering projects
+#     branch: triggering branch name
+#     files: pattern to match triggering filepaths
+- trigger:
+    name: gerrit-trigger-patch-submitted
+    triggers:
+      - gerrit:
+          server-name: '{server}'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: 'ANT'
+                  pattern: '{files}'
+
+- trigger:
+    name: gerrit-trigger-patch-merged
+    triggers:
+      - gerrit:
+          server-name: '{server}'
+          trigger-on:
+            - change-merged-event
+            - comment-added-contains-event:
+                comment-contains-value: 'remerge'
+          projects:
+            - project-compare-type: 'ANT'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: 'ANT'
+                  pattern: '{files}'
+
+# WRAPPERS
+- wrapper:
+    name: build-timeout
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: 360
+          fail: true
+
+- wrapper:
+    # This wrapper is required for all jobs as it configures the wrappers
+    # needed by the eCOMP infra.
+    name: ecomp-infra-wrappers
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: '{build-timeout}'
+          timeout-var: 'BUILD_TIMEOUT'
+          fail: true
+      - ssh-agent-credentials:
+          users:
+            - 'ecomp-jenkins-ssh'
+      - openstack:
+          single-use: true
diff --git a/jjb/global-templates-java.yaml b/jjb/global-templates-java.yaml
new file mode 100644
index 000000000..4ca261f4e
--- /dev/null
+++ b/jjb/global-templates-java.yaml
@@ -0,0 +1,243 @@
+---
+- job-template:
+    # Job template for Java verify jobs
+    #
+    # The purpose of this job template is to run "maven clean install" for
+    # projects using this template.
+    #
+    # Required Variables:
+    #     branch:    git branch (eg. stable/lithium or master)
+
+    name: '{project}-{stream}-verify-java'
+
+    project-type: freestyle
+    concurrent: true
+    node: '{build-node}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: '{build-days-to-keep}'
+          num-to-keep: '{build-num-to-keep}'
+          artifact-days-to-keep: '{build-artifact-days-to-keep}'
+          artifact-num-to-keep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-submitted:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - provide-maven-settings:
+          global-settings-file: 'global-settings'
+          settings-file: '{mvn-settings}'
+      - maven-target:
+          maven-version: 'mvn33'
+          goals: 'clean install'
+          settings: '{mvn-settings}'
+          settings-type: cfp
+          global-settings: 'global-settings'
+          global-settings-type: cfp
+
+    publishers:
+      - ecomp-infra-shiplogs:
+          maven-version: 'mvn33'
+
+- job-template:
+    # Job template for Java verify jobs with POM not at the root
+    #
+    # The purpose of this job template is to run "maven clean install" for
+    # projects using this template.
+    #
+    # Required Variables:
+    #     branch:    git branch (eg. stable/lithium or master)
+    #     pom:       name/location of the pom.xml file relative to the workspace
+    #     pattern:   ant file-path pattern relative to the workspace used to
+    #                trigger the job
+
+    name: '{project}-{stream}-{subproject}-verify-java'
+
+    project-type: freestyle
+    concurrent: true
+    node: '{build-node}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: '{build-days-to-keep}'
+          num-to-keep: '{build-num-to-keep}'
+          artifact-days-to-keep: '{build-artifact-days-to-keep}'
+          artifact-num-to-keep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-submitted-pattern:
+          name: '{project}'
+          branch: '{branch}'
+          pattern: '{pattern}'
+
+    builders:
+      - provide-maven-settings:
+          global-settings-file: 'global-settings'
+          settings-file: '{mvn-settings}'
+      - maven-target:
+          maven-version: 'mvn33'
+          pom: '{pom}'
+          goals: 'clean install'
+          settings: '{mvn-settings}'
+          settings-type: cfp
+          global-settings: 'global-settings'
+          global-settings-type: cfp
+
+    publishers:
+      - ecomp-infra-shiplogs:
+          maven-version: 'mvn33'
+
+- job-template:
+    # Job template for Java merge jobs
+    #
+    # The purpose of this job template is to run "maven clean deploy" for
+    # projects using this template.
+    #
+    # Required Variables:
+    #     branch:    git branch (eg. stable/lithium or master)
+    name: '{project}-{stream}-merge-java'
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: '{build-days-to-keep}'
+          num-to-keep: '{build-num-to-keep}'
+          artifact-days-to-keep: '{build-artifact-days-to-keep}'
+          artifact-num-to-keep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: ''
+          choosing-strategy: 'default'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-merged:
+          name: '{project}'
+          branch: '{branch}'
+
+    builders:
+      - provide-maven-settings:
+          global-settings-file: 'global-settings'
+          settings-file: '{mvn-settings}'
+      - maven-target:
+          maven-version: 'mvn33'
+          # yamllint disable rule:line-length
+          goals: 'clean deploy sonar:sonar -Dsonar.host.url=https://sonar.open-o.org'
+          # yamllint enable
+          settings: '{mvn-settings}'
+          settings-type: cfp
+          global-settings: 'global-settings'
+          global-settings-type: cfp
+
+    publishers:
+      - ecomp-infra-shiplogs:
+          maven-version: 'mvn33'
+
+- job-template:
+    # Job template for Java merge jobs with POM not at the root
+    #
+    # The purpose of this job template is to run "maven clean deploy" for
+    # projects using this template.
+    #
+    # Required Variables:
+    #     branch:    git branch (eg. stable/lithium or master)
+    #     pom:       name/location of the pom.xml file relative to the workspace
+    #     pattern:   ant file-path pattern relative to the workspace used to
+    #                trigger the job
+
+    name: '{project}-{stream}-{subproject}-merge-java'
+
+    project-type: freestyle
+    node: '{build-node}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: '{build-days-to-keep}'
+          num-to-keep: '{build-num-to-keep}'
+          artifact-days-to-keep: '{build-artifact-days-to-keep}'
+          artifact-num-to-keep: '{build-artifact-num-to-keep}'
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: ''
+          choosing-strategy: 'default'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-merged-pattern:
+          name: '{project}'
+          branch: '{branch}'
+          pattern: '{pattern}'
+
+    builders:
+      - provide-maven-settings:
+          global-settings-file: 'global-settings'
+          settings-file: '{mvn-settings}'
+      - maven-target:
+          maven-version: 'mvn33'
+          pom: '{pom}'
+          # yamllint disable rule:line-length
+          goals: 'clean deploy sonar:sonar -Dsonar.host.url=https://sonar.open-o.org'
+          # yamllint enable
+          settings: '{mvn-settings}'
+          settings-type: cfp
+          global-settings: 'global-settings'
+          global-settings-type: cfp
+
+    publishers:
+      - ecomp-infra-shiplogs:
+          maven-version: 'mvn33'
diff --git a/jjb/global-templates-python.yaml b/jjb/global-templates-python.yaml
new file mode 100644
index 000000000..ef4cdeb76
--- /dev/null
+++ b/jjb/global-templates-python.yaml
@@ -0,0 +1,56 @@
+---
+- job-template:
+    # Job template for python verify jobs
+    #
+    # The purpose of this job template is to run tox for projects using this
+    # template.
+    #
+    # Required Variables:
+    #     branch:    git branch (eg. stable/lithium or master)
+    # Optional Variables:
+    #     path:      directory containing the project's tox.ini relative to
+    #                the workspace. The default is the project root.
+    #     pattern:   ant file-path pattern relative to the workspace used to
+    #                trigger the job
+
+    name: '{project}-{stream}-{subproject}-verify-python'
+    path: '$WORKSPACE'
+
+    project-type: freestyle
+    concurrent: true
+    node: '{build-node}'
+
+    parameters:
+      - ecomp-infra-parameters:
+          project: '{project}'
+          branch: '{branch}'
+          refspec: 'refs/heads/{branch}'
+
+    scm:
+      - gerrit-trigger-scm:
+          refspec: '$GERRIT_REFSPEC'
+          choosing-strategy: 'gerrit'
+
+    wrappers:
+      - ecomp-infra-wrappers:
+          build-timeout: '{build-timeout}'
+
+    triggers:
+      - gerrit-trigger-patch-submitted-pattern:
+          name: '{project}'
+          branch: '{branch}'
+          pattern: '{pattern}'
+
+    builders:
+      - shell: |
+          virtualenv $WORKSPACE/venv-tox
+          source $WORKSPACE/venv-tox/bin/activate
+          pip install --upgrade pip
+          pip install --upgrade tox argparse
+          pip freeze
+          cd $WORKSPACE/{path}
+          tox
+
+    publishers:
+      - ecomp-infra-shiplogs:
+          maven-version: 'mvn33'
diff --git a/jjb/include-raw-deploy-archives.sh b/jjb/include-raw-deploy-archives.sh
new file mode 100644
index 000000000..bfe0bcbc7
--- /dev/null
+++ b/jjb/include-raw-deploy-archives.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+set +e  # Do not affect the build result if some part of archiving fails.
+
+ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER"
+[ "$LOGS_SERVER" ] || LOGS_SERVER="https://logs.open-o.org"
+[ "$LOGS_REPO_URL" ] || LOGS_REPO_URL="https://nexus.open-o.org/service/local/repositories/logs"
+
+echo "Build logs: <a href=\"$LOGS_SERVER/$SILO/$ARCHIVES_DIR\">$LOGS_SERVER/$SILO/$ARCHIVES_DIR</a>"
+
+mkdir .archives
+cd .archives/
+
+cat > deploy-archives.xml <<EOF
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>logs</groupId>
+  <artifactId>logs</artifactId>
+  <version>1.0.0</version>
+  <packaging>pom</packaging>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <version>2.8.2</version>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.sonatype.plugins</groupId>
+        <artifactId>maven-upload-plugin</artifactId>
+        <version>0.0.1</version>
+        <executions>
+          <execution>
+            <id>publish-site</id>
+            <phase>deploy</phase>
+            <goals>
+              <goal>upload-file</goal>
+            </goals>
+            <configuration>
+              <serverId>open-o-log-archives</serverId>
+              <repositoryUrl>$LOGS_REPO_URL/content-compressed</repositoryUrl>
+              <file>archives.zip</file>
+              <repositoryPath>$SILO</repositoryPath>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
+EOF
+
+mkdir -p $ARCHIVES_DIR
+mkdir -p $WORKSPACE/archives
+if [ ! -z "${{ARCHIVE_ARTIFACTS}}" ]; then
+    pushd $WORKSPACE
+    shopt -s globstar  # Enable globstar to copy archives
+    archive_artifacts=$(echo ${{ARCHIVE_ARTIFACTS}})
+    for f in $archive_artifacts; do
+        echo "Archiving $f"
+        mkdir -p $WORKSPACE/archives/$(dirname $f)
+        mv $f $WORKSPACE/archives/$f
+    done
+    shopt -u globstar  # Disable globstar once archives are copied
+    popd
+fi
+
+
+# Ignore logging if archives doesn't exist
+mv $WORKSPACE/archives/ $ARCHIVES_DIR > /dev/null 2>&1
+touch $ARCHIVES_DIR/_build-details.txt
+echo "build-url: ${{BUILD_URL}}" >> $ARCHIVES_DIR/_build-details.txt
+env > $ARCHIVES_DIR/_build-enviroment-variables.txt
+
+# capture system info
+touch $ARCHIVES_DIR/_sys-info.txt
+{{
+    echo -e "uname -a:\n `uname -a` \n"
+    echo -e "df -h:\n `df -h` \n"
+    echo -e "free -m:\n `free -m` \n"
+    echo -e "nproc:\n `nproc` \n"
+    echo -e "lscpu:\n `lscpu` \n"
+    echo -e "ip addr:\n  `/sbin/ip addr` \n"
+}} 2>&1 | tee -a $ARCHIVES_DIR/_sys-info.txt
+
+# Magic string used to trim console logs at the appropriate level during wget
+echo "-----END_OF_BUILD-----"
+wget -O $ARCHIVES_DIR/console.log ${{BUILD_URL}}consoleText
+wget -O $ARCHIVES_DIR/console-timestamp.log ${{BUILD_URL}}/timestamps?time=HH:mm:ss\&appendLog
+sed -i '/^-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console.log
+sed -i '/^.*-----END_OF_BUILD-----$/,$d' $ARCHIVES_DIR/console-timestamp.log
+
+gzip $ARCHIVES_DIR/*.txt $ARCHIVES_DIR/*.log
+# find and gzip any 'text' files
+find $ARCHIVES_DIR -type f -print0 \
+                | xargs -0r file \
+                | egrep -e ':.*text.*' \
+                | cut -d: -f1 \
+                | xargs -d'\n' -r gzip
+
+zip -r archives.zip $JENKINS_HOSTNAME/
+du -sh archives.zip