blob: d00fdd64e02a380c3a4435e7b67cf8cff6e0bbd5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
---
- builder:
name: run-rke-ete
builders:
- shell: |
#!/bin/bash
set +x
. $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
. $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
set -x
rm -rf $WORKSPACE/archives
mkdir -p $WORKSPACE/archives
SSH_KEY=~/.ssh/onap_key
ENV_FILE=./env/{lab-name}/onap-oom.env
STACK_NAME={stack-name}
cd $WORKSPACE/deployment/heat/onap-rke/
./scripts/deploy.sh -s $STACK_NAME -d {stack-name}.{lab-name}.onapci.org -i {integration-branch} -o {oom-branch} -q $ENV_FILE
NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
K8S_IP=$(openstack stack output show $STACK_NAME k8s_01_vm_ip -c output_value -f value)
set +x
~/onap-lab-ci/labs/set-dns-record.sh "{stack-name}.{lab-name}" $K8S_IP
set -x
# deploy log and pomba at the end since they're by default disabled in integration-override
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "helm deploy dev local/onap -f ~/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f ~/integration-override.yaml --namespace onap --verbose --set log.enabled=true --set pomba.enabled=true"'
PREV_RESULT=999
for n in $(seq 1 8); do
echo "Wait for pods to be up, $n of 8"
RESULT=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'NAME|Completed|Error|1/1|2/2' | wc -l)
if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then
break
fi
sleep 15m
PREV_RESULT=$RESULT
done
PREV_RESULT=127
for n in $(seq 1 8); do
echo "Wait for HEALTHCHECK, $n of 8"
ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
RESULT=$?
if [[ $RESULT -lt 20 && ( $RESULT -eq 0 || $RESULT -ge $PREV_RESULT ) ]]; then
break
fi
sleep 15m
PREV_RESULT=$RESULT
done
if [ "$ROBOT_POD" == "" ]; then
exit 1
fi
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/healthcheck
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck
# record states of pods and containers
ssh -i $SSH_KEY root@$NFS_IP 'kubectl get pods -n onap'
ssh -i $SSH_KEY root@$NFS_IP "kubectl get pods -n onap -o json" > $WORKSPACE/archives/onap-pods.json
ssh -i $SSH_KEY root@$NFS_IP "/root/integration/deployment/heat/onap-rke/scripts/get-image-tags.sh | tee image-tags.log" > $WORKSPACE/archives/image-tags.log
# demo init
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/demo-k8s.sh onap init"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep demo_init | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/demo-init
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/demo-init
# ete ete
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap ete execscript"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep ete_ete | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/ete
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/ete
# ete instantiate
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiate"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiate | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/instantiate
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiate
# ete portal
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap portal"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep portal | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/portal
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/portal
# ete instantiateDemoVFWCL
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap instantiateDemoVFWCL"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep instantiateDemoVFWCL | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/instantiateDemoVFWCL
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/instantiateDemoVFWCL
# ete sdc-dcae-d
ssh -i $SSH_KEY ubuntu@$NFS_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap sdc-dcae-d"'
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep sdc-dcae-d | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/sdc-dcae-d
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/sdc-dcae-d
# demo vfwclosedloop
PKG_STACK=$(openstack stack list -f value -c "Stack Name" --sort creation_time:desc | grep Vfmodule_Demo_vFWCLvPKG | head -1)
PUBLIC_NET_ID=$(openstack stack show $STACK_NAME -f json | jq -r '.parameters.public_net_id')
PUBLIC_NET_NAME=$(openstack network show $PUBLIC_NET_ID -f value -c name)
PKG_IP=$(openstack stack resource show $PKG_STACK vpg_server_0 -f json | jq -r ".attributes.addresses.$PUBLIC_NET_NAME[0].addr")
echo "/root/oom/kubernetes/robot/demo-k8s.sh onap vfwclosedloop $PKG_IP" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep vfwclosedloop | head -1" | ssh -i $SSH_KEY ubuntu@$NFS_IP sudo su)
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
mkdir -p $WORKSPACE/archives/vfwclosedloop
rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$NFS_IP:/dockerdata-nfs/dev-robot/robot/logs/$LOG_DIR/ $WORKSPACE/archives/vfwclosedloop
cd $WORKSPACE/archives
rebot -N "ONAP CI" --removekeywords wuks --output output.xml --merge $(ls -rt */output.xml)
exit 0
- job-template:
disabled_var:
triggers_var:
name: '{env}-staging-{frequency}'
description: 'Staging deployment to {lab-name} {tenant-name}'
disabled: '{obj:disabled_var}'
project-type: freestyle
lab-name: ''
tenant-name: ''
scm:
- git-integration:
branch: "{integration-branch}"
wrappers:
- timestamps
- timeout:
timeout: 720
fail: true
triggers: '{obj:triggers_var}'
builders:
- run-rke-ete:
stack-name: '{stack-name}'
lab-name: '{lab-name}'
tenant-name: '{tenant-name}'
integration-branch: '{integration-branch}'
oom-branch: '{oom-branch}'
publishers:
- integration-robot
- pods-influxdb
- archive-logs
- trigger-lf-lab-job:
lab-name: '{lab-name}'
- job-template:
disabled_var:
triggers_var:
name: '{env}-release-{frequency}'
description: 'Release deployment to {lab-name} {tenant-name}'
disabled: '{obj:disabled_var}'
project-type: freestyle
lab-name: ''
tenant-name: ''
scm:
- git-integration:
branch: "{integration-branch}"
wrappers:
- timestamps
- timeout:
timeout: 720
fail: true
triggers: '{obj:triggers_var}'
builders:
- run-rke-ete:
stack-name: '{stack-name}'
lab-name: '{lab-name}'
tenant-name: '{tenant-name}'
integration-branch: '{integration-branch}'
oom-branch: '{oom-branch}'
publishers:
- integration-robot
- pods-influxdb
- archive-logs
- trigger-lf-lab-job:
lab-name: '{lab-name}'
- builder:
name: run-cleanup
builders:
- shell: |
#!/bin/bash
set +x
. $WORKSPACE/deployment/heat/onap-rke/env/{lab-name}/{tenant-name}-openrc
. $JENKINS_HOME/onap-lab-ci/labs/{lab-name}-openrc
set -x
rm -rf $WORKSPACE/archives
mkdir -p $WORKSPACE/archives
SSH_KEY=~/.ssh/onap_key
openstack project list -f value | while read line; do
OS_PROJECT_ID=$(echo $line | cut -d' ' -f1)
OS_PROJECT_NAME=$(echo $line | cut -d' ' -f2-)
openstack stack list -f value -c 'Stack Name' | while read STACK_NAME; do
NFS_IP=$(openstack stack output show $STACK_NAME nfs_vm_ip -c output_value -f value)
if [ ! -z "$NFS_IP" ]; then
# timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-log"
# timeout 20 ssh -i $SSH_KEY root@$NFS_IP "helm delete dev-pomba"
timeout 20 ssh -i $SSH_KEY root@$NFS_IP "kubectl top pod | sort --reverse --key 2 --numeric | head -20"
fi
done
done
exit 0
- job-template:
name: '{env}-cleanup'
description: 'Clean up {lab-name} tenants'
project-type: freestyle
lab-name: ''
tenant-name: ''
scm:
- git-integration:
branch: "{integration-branch}"
wrappers:
- timestamps
builders:
- run-cleanup:
lab-name: '{lab-name}'
tenant-name: '{tenant-name}'
publishers:
- archive-logs
|
