diff options
Diffstat (limited to 'deployment/onap-lab-ci/jjb')
| -rw-r--r-- | deployment/onap-lab-ci/jjb/jobs.yaml | 143 | ||||
| -rw-r--r-- | deployment/onap-lab-ci/jjb/macros.yaml | 163 | ||||
| -rw-r--r-- | deployment/onap-lab-ci/jjb/oom-template.yaml | 523 | ||||
| -rw-r--r-- | deployment/onap-lab-ci/jjb/rke-template.yaml | 241 |
4 files changed, 1070 insertions, 0 deletions
diff --git a/deployment/onap-lab-ci/jjb/jobs.yaml b/deployment/onap-lab-ci/jjb/jobs.yaml new file mode 100644 index 000000000..83166e062 --- /dev/null +++ b/deployment/onap-lab-ci/jjb/jobs.yaml @@ -0,0 +1,143 @@ +- project: + name: staging + jobs: + - '{env}-staging-{frequency}' + integration-branch: 'master' + oom-branch: 'staging' + env: + - 'windriver': + lab-name: 'windriver' + tenant-name: 'Integration-Staging-Daily' + stack-name: 'staging' + frequency: 'daily' + disabled_var: false + triggers_var: + - timed: 'H 4 * * *' + - 'windriver-sb00': + lab-name: 'windriver' + tenant-name: 'Integration-SB-00' + stack-name: 'sb00' + frequency: 'manual' + disabled_var: false + - 'windriver-sb01': + lab-name: 'windriver' + tenant-name: 'Integration-SB-01' + stack-name: 'sb01' + frequency: 'manual' + disabled_var: false + - 'windriver-sb02': + lab-name: 'windriver' + tenant-name: 'Integration-SB-02' + stack-name: 'sb02' + frequency: 'manual' + disabled_var: false + - 'windriver-sb03': + lab-name: 'windriver' + tenant-name: 'Integration-SB-03' + stack-name: 'sb03' + frequency: 'manual' + disabled_var: false + - 'windriver-sb04': + lab-name: 'windriver' + tenant-name: 'Integration-SB-04' + stack-name: 'sb04' + frequency: 'manual' + disabled_var: false + - 'windriver-integration-design': + lab-name: 'windriver' + tenant-name: 'Integration-Design' + stack-name: 'design' + frequency: 'manual' + disabled_var: false + - 'windriver-integration-instantiation': + lab-name: 'windriver' + tenant-name: 'Integration-Instantiation' + stack-name: 'instantiation' + frequency: 'manual' + disabled_var: false + - 'windriver-integration-closedloop': + lab-name: 'windriver' + tenant-name: 'Integration-ClosedLoop' + stack-name: 'closedloop' + frequency: 'manual' + disabled_var: false + +- project: + name: release + jobs: + - '{env}-release-{frequency}' + integration-branch: 'master' + oom-branch: 'master' + env: + - 'windriver': + lab-name: 'windriver' + tenant-name: 'Integration-Release-Daily' + stack-name: 'release' + frequency: 'daily' + disabled_var: false + triggers_var: + - timed: 'H 6 * * *' + - 'windriver-longevity': + lab-name: 'windriver' + tenant-name: 'Integration-Longevity' + stack-name: 'long' + frequency: 'manual' + disabled_var: false + - 'windriver-sb00': + lab-name: 'windriver' + tenant-name: 'Integration-SB-00' + stack-name: 'sb00' + frequency: 'manual' + disabled_var: false + - 'windriver-sb01': + lab-name: 'windriver' + tenant-name: 'Integration-SB-01' + stack-name: 'sb01' + frequency: 'manual' + disabled_var: false + - 'windriver-sb02': + lab-name: 'windriver' + tenant-name: 'Integration-SB-02' + stack-name: 'sb02' + frequency: 'manual' + disabled_var: false + - 'windriver-sb03': + lab-name: 'windriver' + tenant-name: 'Integration-SB-03' + stack-name: 'sb03' + frequency: 'manual' + disabled_var: false + - 'windriver-sb04': + lab-name: 'windriver' + tenant-name: 'Integration-SB-04' + stack-name: 'sb04' + frequency: 'manual' + disabled_var: false + +- project: + name: dublin + jobs: + - '{env}-release-{frequency}' + integration-branch: 'master' + oom-branch: 'dublin' + env: + - 'windriver-dublin-sb04': + lab-name: 'windriver' + tenant-name: 'Integration-SB-04' + stack-name: 'dublin' + frequency: 'manual' + disabled_var: false + +- project: + name: stability + jobs: + - '{env}-stability72hr' + - '{env}-vfwclosedloop' + integration-branch: 'master' + env: + - 'windriver-longevity': + lab-name: 'windriver' + tenant-name: 'Integration-Longevity' + stack-name: 'long' + disabled_var: false + diff --git a/deployment/onap-lab-ci/jjb/macros.yaml b/deployment/onap-lab-ci/jjb/macros.yaml new file mode 100644 index 000000000..47d42416b --- /dev/null +++ b/deployment/onap-lab-ci/jjb/macros.yaml @@ -0,0 +1,163 @@ +--- +- scm: + name: git-integration + scm: + - git: + url: 'http://gerrit.onap.org/r/integration' + branches: + - 'origin/{branch}' + wipe-workspace: false + skip-tag: true + timeout: 30 + +- scm: + name: gerrit-trigger-scm + scm: + - git: + url: 'http://gerrit.onap.org/r/$GERRIT_PROJECT' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + choosing-strategy: 'gerrit' + submodule: + recursive: true + +- publisher: + name: integration-robot + publishers: + - robot: + output-path: 'archives' + other-files: '' + unstable-threshold: 60 + pass-threshold: 100 + only-critical: false + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - robot-influxdb + mark-unstable-if-failed: true + +- publisher: + name: pods-influxdb + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - pods-influxdb + mark-unstable-if-failed: true + +- publisher: + name: archive-logs + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - archive-logs + mark-unstable-if-failed: true + +- builder: + name: archive-logs + builders: + - shell: | + #!/bin/bash -x + mkdir -p /var/www/html/logs/$JOB_NAME/$BUILD_NUMBER + curl -s -f "http://localhost:8080/jenkins/job/$JOB_NAME/$BUILD_NUMBER/consoleText" > $WORKSPACE/archives/console.log + curl -s -f "http://localhost:8080/jenkins/job/$JOB_NAME/$BUILD_NUMBER/timestamps/?time=HH:mm:ssZ&appendLog" > $WORKSPACE/archives/console-source-timestamp.log + rsync -avt $WORKSPACE/archives/ /var/www/html/logs/$JOB_NAME/$BUILD_NUMBER + echo + echo "Browse logs at http://onapci.org/logs/$JOB_NAME/$BUILD_NUMBER/" + echo + +- builder: + name: robot-influxdb + builders: + - shell: | + #!/bin/bash -x + # $JENKINS_HOME/onap-lab-ci/scripts/process-robot.sh $WORKSPACE/archives/output.xml $JOB_NAME $BUILD_NUMBER + +- builder: + name: pods-influxdb + builders: + - shell: | + #!/bin/bash -x + # $JENKINS_HOME/onap-lab-ci/scripts/process-pods.sh $WORKSPACE/archives/onap-pods.json $JOB_NAME $BUILD_NUMBER + +- publisher: + name: trigger-lf-lab-job + publishers: + - postbuildscript: + builders: + - role: BOTH + build-on: + - ABORTED + - FAILURE + - NOT_BUILT + - SUCCESS + - UNSTABLE + build-steps: + - trigger-lf-lab-job: + lab-name: '{lab-name}' + mark-unstable-if-failed: false + +- builder: + name: trigger-lf-lab-job + builders: + - shell: | + #!/bin/bash + set +x + LF_JOBS="tlab-oom-daily tlab-oom-staging-daily windriver-oom-daily windriver-oom-staging-daily" + echo $LF_JOBS | grep -q $JOB_NAME + if [ $? -ne 0 ]; then + exit 0 + fi + + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + + set -v + CRUMB=$(curl -s -u "$LF_USERNAME:$LF_PASSWORD" 'https://jenkins.onap.org/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') + curl -s -u "$LF_USERNAME:$LF_PASSWORD" -H "$CRUMB" -X POST "https://jenkins.onap.org/job/lab-$JOB_NAME/buildWithParameters?SRC_BUILD_URL=$BUILD_URL&LOG_DIR_URL=http://onapci.org/logs/$JOB_NAME/$BUILD_NUMBER/" + +- trigger: + name: gerrit-trigger-patch-submitted + triggers: + - gerrit: + server-name: 'gerrit.onap.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '{files}' diff --git a/deployment/onap-lab-ci/jjb/oom-template.yaml b/deployment/onap-lab-ci/jjb/oom-template.yaml new file mode 100644 index 000000000..af8c67a68 --- /dev/null +++ b/deployment/onap-lab-ci/jjb/oom-template.yaml @@ -0,0 +1,523 @@ +--- + +- job-template: + name: '{env}-resilience-healthdist' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + parameters: + - string: + name: POD_TO_DELETE + description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.' + properties: + - build-blocker: + blocking-jobs: + - "{env}-deploy" + - "{env}-healthcheck" + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + STACK_NAME="{stack-name}" + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + ssh-keygen -R $NFS_IP + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su) + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + for n in $(seq 1 180); do + sleep 30 + RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l) + if [[ $? -eq 0 && $RESULT -eq 0 ]]; then + break + fi + done + + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap healthdist"' + retval=$? + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep healthdist | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + if [ "$LOG_DIR" == "" ]; then + exit 1 + fi + + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + + exit 0 + publishers: + - integration-robot + - archive-logs + +- job-template: + name: '{env}-resilience-instantiateDemoVFWCL' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + parameters: + - string: + name: POD_TO_DELETE + description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.' + properties: + - build-blocker: + blocking-jobs: + - "{env}-deploy" + - "{env}-healthcheck" + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + STACK_NAME="{stack-name}" + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + ssh-keygen -R $NFS_IP + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su) + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + for n in $(seq 1 180); do + sleep 30 + RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l) + if [[ $? -eq 0 && $RESULT -eq 0 ]]; then + break + fi + done + + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiateDemoVFWCL"' + retval=$? + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiateDemoVFWCL | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + if [ "$LOG_DIR" == "" ]; then + exit 1 + fi + + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + + # clean up demo stacks + DEMO_STACKS=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_ ) + if [ ! -z "$DEMO_STACKS" ]; then + openstack stack delete -y $DEMO_STACKS + fi + + # clean up vVG VNFs + VVG_SERVERS=$(openstack server list -f value -c Name --sort creation_time:desc | grep vVG_) + if [ ! -z "$VVG_SERVERS" ]; then + openstack server delete $VVG_SERVERS + fi + + exit 0 + publishers: + - integration-robot + - archive-logs + +- job-template: + name: '{env}-resilience-vfwclosedloop' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + parameters: + - string: + name: POD_TO_DELETE + description: 'Name of pod to delete. This string will be grepped against the pod list and used to delete the pod.' + properties: + - build-blocker: + blocking-jobs: + - "{env}-deploy" + - "{env}-healthcheck" + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + STACK_NAME="{stack-name}" + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + ssh-keygen -R $NFS_IP + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su) + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + POD_TO_KILL=$(echo "kubectl -n onap get pods | grep $POD_TO_DELETE | sed 's/ .*//' | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "kubectl -n onap delete pod $POD_TO_KILL" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + for n in $(seq 1 180); do + sleep 30 + RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2|3/3' | wc -l) + if [[ $? -eq 0 && $RESULT -eq 0 ]]; then + break + fi + done + + PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1) + PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id') + PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name) + PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr") + + for n in $(seq 1 10); do + echo "Wait for vfwclosedloop count $n of 10" + echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + RESULT=$? + if [[ $RESULT -eq 0 ]]; then + break + fi + done + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + if [ "$LOG_DIR" == "" ]; then + exit 1 + fi + + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + + exit 0 + publishers: + - integration-robot + - archive-logs + + +- job-template: + disabled_var: + name: '{env}-stability72hr' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + triggers: + - timed: '0 * * * *' + properties: + - build-blocker: + blocking-jobs: + - "{env}-deploy" + - "{env}-healthcheck" + - "{env}-instantiate" + - "{env}-manual" + - "{env}-staging-manual" + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + STACK_NAME="{stack-name}" + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + ssh-keygen -R $NFS_IP + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su) + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap stability72hr"' + retval=$? + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep stability72hr | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + if [ "$LOG_DIR" == "" ]; then + exit 1 + fi + + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + + # clean up demo stacks + DEMO_STACKS=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Ete_ ) + if [ ! -z "$DEMO_STACKS" ]; then + openstack stack delete -y $DEMO_STACKS + fi + + # clean up vVG VNFs + VVG_SERVERS=$(openstack server list -f value -c Name --sort creation_time:desc | grep vVG_) + if [ ! -z "$VVG_SERVERS" ]; then + openstack server delete $VVG_SERVERS + fi + + exit 0 + publishers: + - integration-robot + - archive-logs + +- job-template: + disabled_var: + name: '{env}-vfwclosedloop' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + triggers: + - timed: '0 * * * *' + properties: + - build-blocker: + blocking-jobs: + - "{env}-deploy" + - "{env}-healthcheck" + - "{env}-instantiate" + - "{env}-manual" + - "{env}-staging-manual" + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + STACK_NAME="{stack-name}" + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + ssh-keygen -R $NFS_IP + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + ROBOT_POD=$(echo "kubectl -n onap get pods | grep robot | sed 's/ .*//'" | ssh -i $SSH_KEY -o StrictHostKeychecking=no ubuntu@$NFS_IP sudo su) + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1) + PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id') + PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name) + PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr") + + + echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + retval=$? + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + if [ "$LOG_DIR" == "" ]; then + exit 1 + fi + + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ + + echo "kubectl top nodes" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + echo "kubectl get pods -n onap | grep -vE 'Completed|Error|1/1|2/2|3/3'" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + + exit 0 + publishers: + - integration-robot + - archive-logs + + +- job-template: + name: 'oom-verify' + project-type: freestyle + parameters: + - string: + name: GERRIT_PROJECT + default: 'oom' + description: 'GERRIT_PROJECT parameter if not given by trigger' + - string: + name: GERRIT_BRANCH + default: '{branch}' + description: 'JJB configured GERRIT_BRANCH parameter' + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{branch}' + description: 'GERRIT_REFSPEC parameter if not given by trigger' + scm: + - gerrit-trigger-scm + triggers: + - gerrit: + server-name: 'gerrit.onap.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + projects: + - project-compare-type: 'ANT' + project-pattern: 'oom' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: 'kubernetes/**/*.yaml' + - project-compare-type: 'ANT' + project-pattern: '*/oom' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '**/*.yaml' + wrappers: + - timestamps + - timeout: + timeout: 720 + fail: true + builders: + - shell: | + #!/bin/bash + + set +e + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + NEXUS_RELEASE_PREFIX="https://nexus3.onap.org/repository/docker.release/v2" + RELEASE_TAGS_PATH="/tmp/onap-docker-release" + err=0 + + # if no files changed, will scan all files + CHANGED_FILES="" + if [ $(git rev-parse HEAD) != $(git rev-parse origin/master) ]; then + CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r HEAD) + fi + IMAGES_FILE=$WORKSPACE/archives/images.txt + rgrep --exclude-dir=pnda -n -E ':\s*onap/.*:.*' $CHANGED_FILES | awk '{{$1=$1}};1' | sort > $IMAGES_FILE + + set +x + while read line; do + location=$(echo $line | cut -d: -f-2) + image_tag=$(echo $line | cut -d: -f3- | awk '{{$1=$1}};1' | cut -d' ' -f2) + image=$(echo $image_tag | cut -d : -f 1) + tag=$(echo $image_tag | cut -d : -f 2) + + case $tag in + *STAGING*) + echo "[ERROR] $location: $image:$tag not released" + (( err++ )) + ;; + *SNAPSHOT*) + echo "[ERROR] $location: $image:$tag not released" + (( err++ )) + ;; + *latest*) + echo "[ERROR] $location: $image:$tag not released" + (( err++ )) + ;; + *) + TAGS_FILE=$RELEASE_TAGS_PATH/$image/tags.txt + mkdir -p $RELEASE_TAGS_PATH/$image + touch $TAGS_FILE + grep -q "^$tag\$" $TAGS_FILE + if [ $? -ne 0 ]; then + # not found; download latest + curl -s $NEXUS_RELEASE_PREFIX/$image/tags/list | jq -r '.tags[]' > $TAGS_FILE 2> /dev/null + grep -q "^$tag\$" $TAGS_FILE + if [ $? -ne 0 ]; then + echo "[ERROR] $location: $image:$tag not released" + (( err++ )) + fi + fi + ;; + esac + done < $IMAGES_FILE + echo $err unreleased images found. + exit $err + + publishers: + - archive-logs diff --git a/deployment/onap-lab-ci/jjb/rke-template.yaml b/deployment/onap-lab-ci/jjb/rke-template.yaml new file mode 100644 index 000000000..d00fdd64e --- /dev/null +++ b/deployment/onap-lab-ci/jjb/rke-template.yaml @@ -0,0 +1,241 @@ +--- +- builder: + name: run-rke-ete + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + ENV_FILE=./env/{lab-name}/onap-oom.env + STACK_NAME={stack-name} + + cd $WORKSPACE/deployment/heat/onap-rke/ + ./scripts/deploy.sh -s $STACK_NAME -d {stack-name}.{lab-name}.onapci.org -i {integration-branch} -o {oom-branch} -q $ENV_FILE + + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value) + + set +x + ~/onap-lab-ci/labs/set-dns-record.sh "{stack-name}.{lab-name}" $K8S_IP + set -x + + # deploy log and pomba at the end since they're by default disabled in integration-override + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "helm deploy dev local/onap -f ~/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f ~/integration-override.yaml --namespace onap --verbose --set log.enabled=true --set pomba.enabled=true"' + + PREV_RESULT=999 + for n in $(seq 1 8); do + echo "Wait for pods to be up, $n of 8" + RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2' | wc -l) + if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then + break + fi + sleep 15m + PREV_RESULT=$RESULT + done + + PREV_RESULT=127 + for n in $(seq 1 8); do + echo "Wait for HEALTHCHECK, $n of 8" + ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//') + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"' + RESULT=$? + if [[ $RESULT -lt 20 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then + break + fi + sleep 15m + PREV_RESULT=$RESULT + done + if [ "$ROBOT_POD" == "" ]; then + exit 1 + fi + + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/healthcheck + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck + + # record states of pods and containers + ssh -i $SSH_KEY root@$NFS_IP 'kubectl get pods -n onap' + ssh -i $SSH_KEY root@$NFS_IP "kubectl get pods -n onap -o json" > $WORKSPACE/archives/onap-pods.json + ssh -i $SSH_KEY root@$NFS_IP "/root/integration/deployment/heat/onap-rke/scripts/get-image-tags.sh | tee image-tags.log" > $WORKSPACE/archives/image-tags.log + + # demo init + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/demo-k8s.sh onap init"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep demo_init | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/demo-init + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/demo-init + + # ete ete + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap ete execscript"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep ete_ete | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/ete + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ete + + # ete instantiate + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiate"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiate | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/instantiate + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiate + + # ete portal + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap portal"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep portal | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/portal + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/portal + + # ete instantiateDemoVFWCL + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiateDemoVFWCL"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiateDemoVFWCL | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/instantiateDemoVFWCL + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiateDemoVFWCL + + # ete sdc-dcae-d + ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap sdc-dcae-d"' + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep sdc-dcae-d | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/sdc-dcae-d + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/sdc-dcae-d + + + # demo vfwclosedloop + PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1) + PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id') + PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name) + PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr") + echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su + LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su) + echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/" + mkdir -p $WORKSPACE/archives/vfwclosedloop + rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/vfwclosedloop + + + cd $WORKSPACE/archives + rebot -N "ONAP CI" --removekeywords wuks --output output.xml --merge $(ls -rt */output.xml) + + exit 0 + +- job-template: + disabled_var: + triggers_var: + name: '{env}-staging-{frequency}' + description: 'Staging deployment to {lab-name} {tenant-name}' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + - timeout: + timeout: 720 + fail: true + triggers: '{obj:triggers_var}' + builders: + - run-rke-ete: + stack-name: '{stack-name}' + lab-name: '{lab-name}' + tenant-name: '{tenant-name}' + integration-branch: '{integration-branch}' + oom-branch: '{oom-branch}' + publishers: + - integration-robot + - pods-influxdb + - archive-logs + - trigger-lf-lab-job: + lab-name: '{lab-name}' + +- job-template: + disabled_var: + triggers_var: + name: '{env}-release-{frequency}' + description: 'Release deployment to {lab-name} {tenant-name}' + disabled: '{obj:disabled_var}' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + - timeout: + timeout: 720 + fail: true + triggers: '{obj:triggers_var}' + builders: + - run-rke-ete: + stack-name: '{stack-name}' + lab-name: '{lab-name}' + tenant-name: '{tenant-name}' + integration-branch: '{integration-branch}' + oom-branch: '{oom-branch}' + publishers: + - integration-robot + - pods-influxdb + - archive-logs + - trigger-lf-lab-job: + lab-name: '{lab-name}' + +- builder: + name: run-cleanup + builders: + - shell: | + #!/bin/bash + set +x + . $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc + . $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc + + set -x + rm -rf $WORKSPACE/archives + mkdir -p $WORKSPACE/archives + + SSH_KEY=~/.ssh/onap_key + + openstack project list -f value | while read line; do + OS_PROJECT_ID=$(echo $line | cut -d' ' -f1) + OS_PROJECT_NAME=$(echo $line | cut -d' ' -f2-) + + openstack stack list -f value -c 'Stack Name' | while read STACK_NAME; do + NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value) + if [ ! -z "$NFS_IP" ]; then + # timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-log" + # timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-pomba" + timeout 20 ssh -i $SSH_KEY root@$NFS_IP "kubectl top pod | sort --reverse --key 2 --numeric | head -20" + fi + done + done + + exit 0 + +- job-template: + name: '{env}-cleanup' + description: 'Clean up {lab-name} tenants' + project-type: freestyle + lab-name: '' + tenant-name: '' + scm: + - git-integration: + branch: "{integration-branch}" + wrappers: + - timestamps + builders: + - run-cleanup: + lab-name: '{lab-name}' + tenant-name: '{tenant-name}' + publishers: + - archive-logs |