aboutsummaryrefslogtreecommitdiffstats
path: root/deployment/onap-lab-ci/jjb
diff options
context:
space:
mode:
Diffstat (limited to 'deployment/onap-lab-ci/jjb')
-rw-r--r--deployment/onap-lab-ci/jjb/jobs.yaml143
-rw-r--r--deployment/onap-lab-ci/jjb/macros.yaml163
-rw-r--r--deployment/onap-lab-ci/jjb/oom-template.yaml523
-rw-r--r--deployment/onap-lab-ci/jjb/rke-template.yaml241
4 files changed, 1070 insertions, 0 deletions
diff --git a/deployment/onap-lab-ci/jjb/jobs.yaml b/deployment/onap-lab-ci/jjb/jobs.yaml
new file mode 100644
index 000000000..83166e062
--- /dev/null
+++ b/deployment/onap-lab-ci/jjb/jobs.yaml
@@ -0,0 +1,143 @@
+- project:
+ name: staging
+ jobs:
+ - '{env}-staging-{frequency}'
+ integration-branch: 'master'
+ oom-branch: 'staging'
+ env:
+ - 'windriver':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Staging-Daily'
+ stack-name: 'staging'
+ frequency: 'daily'
+ disabled_var: false
+ triggers_var:
+ - timed: 'H 4 * * *'
+ - 'windriver-sb00':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-00'
+ stack-name: 'sb00'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb01':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-01'
+ stack-name: 'sb01'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb02':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-02'
+ stack-name: 'sb02'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb03':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-03'
+ stack-name: 'sb03'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb04':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-04'
+ stack-name: 'sb04'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-integration-design':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Design'
+ stack-name: 'design'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-integration-instantiation':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Instantiation'
+ stack-name: 'instantiation'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-integration-closedloop':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-ClosedLoop'
+ stack-name: 'closedloop'
+ frequency: 'manual'
+ disabled_var: false
+
+- project:
+ name: release
+ jobs:
+ - '{env}-release-{frequency}'
+ integration-branch: 'master'
+ oom-branch: 'master'
+ env:
+ - 'windriver':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Release-Daily'
+ stack-name: 'release'
+ frequency: 'daily'
+ disabled_var: false
+ triggers_var:
+ - timed: 'H 6 * * *'
+ - 'windriver-longevity':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Longevity'
+ stack-name: 'long'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb00':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-00'
+ stack-name: 'sb00'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb01':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-01'
+ stack-name: 'sb01'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb02':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-02'
+ stack-name: 'sb02'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb03':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-03'
+ stack-name: 'sb03'
+ frequency: 'manual'
+ disabled_var: false
+ - 'windriver-sb04':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-04'
+ stack-name: 'sb04'
+ frequency: 'manual'
+ disabled_var: false
+
+- project:
+ name: dublin
+ jobs:
+ - '{env}-release-{frequency}'
+ integration-branch: 'master'
+ oom-branch: 'dublin'
+ env:
+ - 'windriver-dublin-sb04':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-SB-04'
+ stack-name: 'dublin'
+ frequency: 'manual'
+ disabled_var: false
+
+- project:
+ name: stability
+ jobs:
+ - '{env}-stability72hr'
+ - '{env}-vfwclosedloop'
+ integration-branch: 'master'
+ env:
+ - 'windriver-longevity':
+ lab-name: 'windriver'
+ tenant-name: 'Integration-Longevity'
+ stack-name: 'long'
+ disabled_var: false
+
diff --git a/deployment/onap-lab-ci/jjb/macros.yaml b/deployment/onap-lab-ci/jjb/macros.yaml
new file mode 100644
index 000000000..47d42416b
--- /dev/null
+++ b/deployment/onap-lab-ci/jjb/macros.yaml
@@ -0,0 +1,163 @@
+---
+- scm:
+ name: git-integration
+ scm:
+ - git:
+ url: 'http://gerrit.onap.org/r/integration'
+ branches:
+ - 'origin/{branch}'
+ wipe-workspace: false
+ skip-tag: true
+ timeout: 30
+
+- scm:
+ name: gerrit-trigger-scm
+ scm:
+ - git:
+ url: 'http://gerrit.onap.org/r/$GERRIT_PROJECT'
+ refspec: '$GERRIT_REFSPEC'
+ branches:
+ - 'origin/$GERRIT_BRANCH'
+ skip-tag: true
+ choosing-strategy: 'gerrit'
+ submodule:
+ recursive: true
+
+- publisher:
+ name: integration-robot
+ publishers:
+ - robot:
+ output-path: 'archives'
+ other-files: ''
+ unstable-threshold: 60
+ pass-threshold: 100
+ only-critical: false
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - NOT_BUILT
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - robot-influxdb
+ mark-unstable-if-failed: true
+
+- publisher:
+ name: pods-influxdb
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - NOT_BUILT
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - pods-influxdb
+ mark-unstable-if-failed: true
+
+- publisher:
+ name: archive-logs
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - NOT_BUILT
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - archive-logs
+ mark-unstable-if-failed: true
+
+- builder:
+ name: archive-logs
+ builders:
+ - shell: |
+ #!/bin/bash -x
+ mkdir -p /var/www/html/logs/$JOB_NAME/$BUILD_NUMBER
+ curl -s -f "http://localhost:8080/jenkins/job/$JOB_NAME/$BUILD_NUMBER/consoleText" > $WORKSPACE/archives/console.log
+ curl -s -f "http://localhost:8080/jenkins/job/$JOB_NAME/$BUILD_NUMBER/timestamps/?time=HH:mm:ssZ&appendLog" > $WORKSPACE/archives/console-source-timestamp.log
+ rsync -avt $WORKSPACE/archives/ /var/www/html/logs/$JOB_NAME/$BUILD_NUMBER
+ echo
+ echo "Browse logs at http://onapci.org/logs/$JOB_NAME/$BUILD_NUMBER/"
+ echo
+
+- builder:
+ name: robot-influxdb
+ builders:
+ - shell: |
+ #!/bin/bash -x
+ # $JENKINS_HOME/onap-lab-ci/scripts/process-robot.sh $WORKSPACE/archives/output.xml $JOB_NAME $BUILD_NUMBER
+
+- builder:
+ name: pods-influxdb
+ builders:
+ - shell: |
+ #!/bin/bash -x
+ # $JENKINS_HOME/onap-lab-ci/scripts/process-pods.sh $WORKSPACE/archives/onap-pods.json $JOB_NAME $BUILD_NUMBER
+
+- publisher:
+ name: trigger-lf-lab-job
+ publishers:
+ - postbuildscript:
+ builders:
+ - role: BOTH
+ build-on:
+ - ABORTED
+ - FAILURE
+ - NOT_BUILT
+ - SUCCESS
+ - UNSTABLE
+ build-steps:
+ - trigger-lf-lab-job:
+ lab-name: '{lab-name}'
+ mark-unstable-if-failed: false
+
+- builder:
+ name: trigger-lf-lab-job
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ LF_JOBS="tlab-oom-daily tlab-oom-staging-daily windriver-oom-daily windriver-oom-staging-daily"
+ echo $LF_JOBS | grep -q $JOB_NAME
+ if [ $? -ne 0 ]; then
+ exit 0
+ fi
+
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+
+ set -v
+ CRUMB=$(curl -s -u "$LF_USERNAME:$LF_PASSWORD" 'https://jenkins.onap.org/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)')
+ curl -s -u "$LF_USERNAME:$LF_PASSWORD" -H "$CRUMB" -X POST "https://jenkins.onap.org/job/lab-$JOB_NAME/buildWithParameters?SRC_BUILD_URL=$BUILD_URL&LOG_DIR_URL=http://onapci.org/logs/$JOB_NAME/$BUILD_NUMBER/"
+
+- trigger:
+ name: gerrit-trigger-patch-submitted
+ triggers:
+ - gerrit:
+ server-name: 'gerrit.onap.org'
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: 'false'
+ exclude-trivial-rebase: 'false'
+ exclude-no-code-change: 'false'
+ - draft-published-event
+ - comment-added-contains-event:
+ comment-contains-value: 'recheck'
+ projects:
+ - project-compare-type: 'ANT'
+ project-pattern: '{project}'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**/{branch}'
+ file-paths:
+ - compare-type: 'ANT'
+ pattern: '{files}'
diff --git a/deployment/onap-lab-ci/jjb/oom-template.yaml b/deployment/onap-lab-ci/jjb/oom-template.yaml
new file mode 100644
index 000000000..af8c67a68
--- /dev/null
+++ b/deployment/onap-lab-ci/jjb/oom-template.yaml
@@ -0,0 +1,523 @@
+---
+
+- job-template:
+ name: '{env}-resilience-healthdist'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ parameters:
+ - string:
+ name: POD_TO_DELETE
+ description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.'
+ properties:
+ - build-blocker:
+ blocking-jobs:
+ - "{env}-deploy"
+ - "{env}-healthcheck"
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ STACK_NAME="{stack-name}"
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+ ssh-keygen -R $NFS_IP
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su)
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ for n in $(seq 1 180); do
+ sleep 30
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l)
+ if [[ $? -eq 0 && $RESULT -eq 0 ]]; then
+ break
+ fi
+ done
+
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap healthdist"'
+ retval=$?
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep healthdist | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ if [ "$LOG_DIR" == "" ]; then
+ exit 1
+ fi
+
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+
+ exit 0
+ publishers:
+ - integration-robot
+ - archive-logs
+
+- job-template:
+ name: '{env}-resilience-instantiateDemoVFWCL'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ parameters:
+ - string:
+ name: POD_TO_DELETE
+ description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.'
+ properties:
+ - build-blocker:
+ blocking-jobs:
+ - "{env}-deploy"
+ - "{env}-healthcheck"
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ STACK_NAME="{stack-name}"
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+ ssh-keygen -R $NFS_IP
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su)
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ for n in $(seq 1 180); do
+ sleep 30
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l)
+ if [[ $? -eq 0 && $RESULT -eq 0 ]]; then
+ break
+ fi
+ done
+
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiateDemoVFWCL"'
+ retval=$?
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiateDemoVFWCL | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ if [ "$LOG_DIR" == "" ]; then
+ exit 1
+ fi
+
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+
+ # clean up demo stacks
+ DEMO_STACKS=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_ )
+ if [ ! -z "$DEMO_STACKS" ]; then
+ openstack stack delete -y $DEMO_STACKS
+ fi
+
+ # clean up vVG VNFs
+ VVG_SERVERS=$(openstack server list -f value -c Name --sort creation_time:desc | grep vVG_)
+ if [ ! -z "$VVG_SERVERS" ]; then
+ openstack server delete $VVG_SERVERS
+ fi
+
+ exit 0
+ publishers:
+ - integration-robot
+ - archive-logs
+
+- job-template:
+ name: '{env}-resilience-vfwclosedloop'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ parameters:
+ - string:
+ name: POD_TO_DELETE
+ description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.'
+ properties:
+ - build-blocker:
+ blocking-jobs:
+ - "{env}-deploy"
+ - "{env}-healthcheck"
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ STACK_NAME="{stack-name}"
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+ ssh-keygen -R $NFS_IP
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su)
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ for n in $(seq 1 180); do
+ sleep 30
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l)
+ if [[ $? -eq 0 && $RESULT -eq 0 ]]; then
+ break
+ fi
+ done
+
+ PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1)
+ PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id')
+ PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name)
+ PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr")
+
+ for n in $(seq 1 10); do
+ echo "Wait for vfwclosedloop count $n of 10"
+ echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ RESULT=$?
+ if [[ $RESULT -eq 0 ]]; then
+ break
+ fi
+ done
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ if [ "$LOG_DIR" == "" ]; then
+ exit 1
+ fi
+
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+
+ exit 0
+ publishers:
+ - integration-robot
+ - archive-logs
+
+
+- job-template:
+ disabled_var:
+ name: '{env}-stability72hr'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ triggers:
+ - timed: '0 * * * *'
+ properties:
+ - build-blocker:
+ blocking-jobs:
+ - "{env}-deploy"
+ - "{env}-healthcheck"
+ - "{env}-instantiate"
+ - "{env}-manual"
+ - "{env}-staging-manual"
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ STACK_NAME="{stack-name}"
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+ ssh-keygen -R $NFS_IP
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su)
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap stability72hr"'
+ retval=$?
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep stability72hr | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ if [ "$LOG_DIR" == "" ]; then
+ exit 1
+ fi
+
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+
+ # clean up demo stacks
+ DEMO_STACKS=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Ete_ )
+ if [ ! -z "$DEMO_STACKS" ]; then
+ openstack stack delete -y $DEMO_STACKS
+ fi
+
+ # clean up vVG VNFs
+ VVG_SERVERS=$(openstack server list -f value -c Name --sort creation_time:desc | grep vVG_)
+ if [ ! -z "$VVG_SERVERS" ]; then
+ openstack server delete $VVG_SERVERS
+ fi
+
+ exit 0
+ publishers:
+ - integration-robot
+ - archive-logs
+
+- job-template:
+ disabled_var:
+ name: '{env}-vfwclosedloop'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ triggers:
+ - timed: '0 * * * *'
+ properties:
+ - build-blocker:
+ blocking-jobs:
+ - "{env}-deploy"
+ - "{env}-healthcheck"
+ - "{env}-instantiate"
+ - "{env}-manual"
+ - "{env}-staging-manual"
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ STACK_NAME="{stack-name}"
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+ ssh-keygen -R $NFS_IP
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su)
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1)
+ PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id')
+ PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name)
+ PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr")
+
+
+ echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ retval=$?
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ if [ "$LOG_DIR" == "" ]; then
+ exit 1
+ fi
+
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
+
+ echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+
+ exit 0
+ publishers:
+ - integration-robot
+ - archive-logs
+
+
+- job-template:
+ name: 'oom-verify'
+ project-type: freestyle
+ parameters:
+ - string:
+ name: GERRIT_PROJECT
+ default: 'oom'
+ description: 'GERRIT_PROJECT parameter if not given by trigger'
+ - string:
+ name: GERRIT_BRANCH
+ default: '{branch}'
+ description: 'JJB configured GERRIT_BRANCH parameter'
+ - string:
+ name: GERRIT_REFSPEC
+ default: 'refs/heads/{branch}'
+ description: 'GERRIT_REFSPEC parameter if not given by trigger'
+ scm:
+ - gerrit-trigger-scm
+ triggers:
+ - gerrit:
+ server-name: 'gerrit.onap.org'
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: 'false'
+ exclude-trivial-rebase: 'false'
+ exclude-no-code-change: 'false'
+ - draft-published-event
+ - comment-added-contains-event:
+ comment-contains-value: 'recheck'
+ projects:
+ - project-compare-type: 'ANT'
+ project-pattern: 'oom'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**/{branch}'
+ file-paths:
+ - compare-type: 'ANT'
+ pattern: 'kubernetes/**/*.yaml'
+ - project-compare-type: 'ANT'
+ project-pattern: '*/oom'
+ branches:
+ - branch-compare-type: 'ANT'
+ branch-pattern: '**/{branch}'
+ file-paths:
+ - compare-type: 'ANT'
+ pattern: '**/*.yaml'
+ wrappers:
+ - timestamps
+ - timeout:
+ timeout: 720
+ fail: true
+ builders:
+ - shell: |
+ #!/bin/bash
+
+ set +e
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ NEXUS_RELEASE_PREFIX="https://nexus3.onap.org/repository/docker.release/v2"
+ RELEASE_TAGS_PATH="/tmp/onap-docker-release"
+ err=0
+
+ # if no files changed, will scan all files
+ CHANGED_FILES=""
+ if [ $(git rev-parse HEAD) != $(git rev-parse origin/master) ]; then
+ CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r HEAD)
+ fi
+ IMAGES_FILE=$WORKSPACE/archives/images.txt
+ rgrep --exclude-dir=pnda -n -E ':\s*onap/.*:.*' $CHANGED_FILES | awk '{{$1=$1}};1' | sort > $IMAGES_FILE
+
+ set +x
+ while read line; do
+ location=$(echo $line | cut -d: -f-2)
+ image_tag=$(echo $line | cut -d: -f3- | awk '{{$1=$1}};1' | cut -d' ' -f2)
+ image=$(echo $image_tag | cut -d : -f 1)
+ tag=$(echo $image_tag | cut -d : -f 2)
+
+ case $tag in
+ *STAGING*)
+ echo "[ERROR] $location: $image:$tag not released"
+ (( err++ ))
+ ;;
+ *SNAPSHOT*)
+ echo "[ERROR] $location: $image:$tag not released"
+ (( err++ ))
+ ;;
+ *latest*)
+ echo "[ERROR] $location: $image:$tag not released"
+ (( err++ ))
+ ;;
+ *)
+ TAGS_FILE=$RELEASE_TAGS_PATH/$image/tags.txt
+ mkdir -p $RELEASE_TAGS_PATH/$image
+ touch $TAGS_FILE
+ grep -q "^$tag\$" $TAGS_FILE
+ if [ $? -ne 0 ]; then
+ # not found; download latest
+ curl -s $NEXUS_RELEASE_PREFIX/$image/tags/list | jq -r '.tags[]' > $TAGS_FILE 2> /dev/null
+ grep -q "^$tag\$" $TAGS_FILE
+ if [ $? -ne 0 ]; then
+ echo "[ERROR] $location: $image:$tag not released"
+ (( err++ ))
+ fi
+ fi
+ ;;
+ esac
+ done < $IMAGES_FILE
+ echo $err unreleased images found.
+ exit $err
+
+ publishers:
+ - archive-logs
diff --git a/deployment/onap-lab-ci/jjb/rke-template.yaml b/deployment/onap-lab-ci/jjb/rke-template.yaml
new file mode 100644
index 000000000..d00fdd64e
--- /dev/null
+++ b/deployment/onap-lab-ci/jjb/rke-template.yaml
@@ -0,0 +1,241 @@
+---
+- builder:
+ name: run-rke-ete
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+ source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+ ENV_FILE=./env/{lab-name}/onap-oom.env
+ STACK_NAME={stack-name}
+
+ cd $WORKSPACE/deployment/heat/onap-rke/
+ ./scripts/deploy.sh -s $STACK_NAME -d {stack-name}.{lab-name}.onapci.org -i {integration-branch} -o {oom-branch} -q $ENV_FILE
+
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
+
+ set +x
+ ~/onap-lab-ci/labs/set-dns-record.sh "{stack-name}.{lab-name}" $K8S_IP
+ set -x
+
+ # deploy log and pomba at the end since they're by default disabled in integration-override
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "helm deploy dev local/onap -f ~/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f ~/integration-override.yaml --namespace onap --verbose --set log.enabled=true --set pomba.enabled=true"'
+
+ PREV_RESULT=999
+ for n in $(seq 1 8); do
+ echo "Wait for pods to be up, $n of 8"
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2' | wc -l)
+ if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
+ done
+
+ PREV_RESULT=127
+ for n in $(seq 1 8); do
+ echo "Wait for HEALTHCHECK, $n of 8"
+ ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
+ RESULT=$?
+ if [[ $RESULT -lt 20 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
+ done
+ if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+ fi
+
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/healthcheck
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck
+
+ # record states of pods and containers
+ ssh -i $SSH_KEY root@$NFS_IP 'kubectl get pods -n onap'
+ ssh -i $SSH_KEY root@$NFS_IP "kubectl get pods -n onap -o json" > $WORKSPACE/archives/onap-pods.json
+ ssh -i $SSH_KEY root@$NFS_IP "/root/integration/deployment/heat/onap-rke/scripts/get-image-tags.sh | tee image-tags.log" > $WORKSPACE/archives/image-tags.log
+
+ # demo init
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/demo-k8s.sh onap init"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep demo_init | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/demo-init
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/demo-init
+
+ # ete ete
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap ete execscript"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep ete_ete | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/ete
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ete
+
+ # ete instantiate
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiate"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiate | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/instantiate
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiate
+
+ # ete portal
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap portal"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep portal | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/portal
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/portal
+
+ # ete instantiateDemoVFWCL
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiateDemoVFWCL"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiateDemoVFWCL | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/instantiateDemoVFWCL
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiateDemoVFWCL
+
+ # ete sdc-dcae-d
+ ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap sdc-dcae-d"'
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep sdc-dcae-d | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/sdc-dcae-d
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/sdc-dcae-d
+
+
+ # demo vfwclosedloop
+ PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1)
+ PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id')
+ PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name)
+ PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr")
+ echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
+ LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
+ echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+ mkdir -p $WORKSPACE/archives/vfwclosedloop
+ rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/vfwclosedloop
+
+
+ cd $WORKSPACE/archives
+ rebot -N "ONAP CI" --removekeywords wuks --output output.xml --merge $(ls -rt */output.xml)
+
+ exit 0
+
+- job-template:
+ disabled_var:
+ triggers_var:
+ name: '{env}-staging-{frequency}'
+ description: 'Staging deployment to {lab-name} {tenant-name}'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ - timeout:
+ timeout: 720
+ fail: true
+ triggers: '{obj:triggers_var}'
+ builders:
+ - run-rke-ete:
+ stack-name: '{stack-name}'
+ lab-name: '{lab-name}'
+ tenant-name: '{tenant-name}'
+ integration-branch: '{integration-branch}'
+ oom-branch: '{oom-branch}'
+ publishers:
+ - integration-robot
+ - pods-influxdb
+ - archive-logs
+ - trigger-lf-lab-job:
+ lab-name: '{lab-name}'
+
+- job-template:
+ disabled_var:
+ triggers_var:
+ name: '{env}-release-{frequency}'
+ description: 'Release deployment to {lab-name} {tenant-name}'
+ disabled: '{obj:disabled_var}'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ - timeout:
+ timeout: 720
+ fail: true
+ triggers: '{obj:triggers_var}'
+ builders:
+ - run-rke-ete:
+ stack-name: '{stack-name}'
+ lab-name: '{lab-name}'
+ tenant-name: '{tenant-name}'
+ integration-branch: '{integration-branch}'
+ oom-branch: '{oom-branch}'
+ publishers:
+ - integration-robot
+ - pods-influxdb
+ - archive-logs
+ - trigger-lf-lab-job:
+ lab-name: '{lab-name}'
+
+- builder:
+ name: run-cleanup
+ builders:
+ - shell: |
+ #!/bin/bash
+ set +x
+ . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
+ . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
+
+ set -x
+ rm -rf $WORKSPACE/archives
+ mkdir -p $WORKSPACE/archives
+
+ SSH_KEY=~/.ssh/onap_key
+
+ openstack project list -f value | while read line; do
+ OS_PROJECT_ID=$(echo $line | cut -d' ' -f1)
+ OS_PROJECT_NAME=$(echo $line | cut -d' ' -f2-)
+
+ openstack stack list -f value -c 'Stack Name' | while read STACK_NAME; do
+ NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
+ if [ ! -z "$NFS_IP" ]; then
+ # timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-log"
+ # timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-pomba"
+ timeout 20 ssh -i $SSH_KEY root@$NFS_IP "kubectl top pod | sort --reverse --key 2 --numeric | head -20"
+ fi
+ done
+ done
+
+ exit 0
+
+- job-template:
+ name: '{env}-cleanup'
+ description: 'Clean up {lab-name} tenants'
+ project-type: freestyle
+ lab-name: ''
+ tenant-name: ''
+ scm:
+ - git-integration:
+ branch: "{integration-branch}"
+ wrappers:
+ - timestamps
+ builders:
+ - run-cleanup:
+ lab-name: '{lab-name}'
+ tenant-name: '{tenant-name}'
+ publishers:
+ - archive-logs