1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
#!/usr/bin/env python
"""
This script attempts to probe an ONAP deployment in a Kubernetes cluster on an OpenStack stack
and extract all pods and corresponding docker image versions along with some of the tool versions utilised.
"""
import os
import sys
import subprocess
import argparse
import logging
OPENSTACK_GET_SERVER_IPS = 'openstack server list --name "^(%s-).*" -c Name -c Networks -f value --sort-column Name'
SSH_CMD_TEMPLATE = 'ssh -o StrictHostKeychecking=no -i %s ubuntu@%s "sudo su -c \\"%s\\""'
KUBECTL_GET_ALL_POD_IMAGES_AND_SHAS = 'kubectl get pods --all-namespaces -o=jsonpath=\'{range .items[*]}' \
'{\\\\\\"\\n\\\\\\"}{.metadata.name}{\\\\\\":::\\\\\\"}' \
'{range .status.containerStatuses[*]}{.image}{\\\\\\"___\\\\\\"}' \
'{.imageID}{\\\\\\" \\\\\\"}{end}{end}{\\\\\\"\\n\\\\\\"}\''
DOCKER_INSPECT = 'docker inspect --format=\'{{index .RepoTags 0}}{{\\\\\\" \\\\\\"}}{{index .RepoDigests 0}}\' ' \
'\$(docker images -q | uniq| tr \'\n\' \' \')'
KUBECTL_VERSION = 'kubectl version'
DOCKER_VERSION = 'docker --version'
local_registry = ""
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
file_log = logging.FileHandler(filename='onap-probe-report.txt', mode='w')
file_log.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
file_log.setFormatter(formatter)
logging.getLogger('').addHandler(file_log)
class CommandResult(object):
def __init__(self, exit_code, stdout, stderr):
self.exit_code = exit_code
self.stdout = stdout
self.stderr = stderr
def run_command_or_exit(command, message=""):
if message:
logging.debug(message)
logging.debug('cmd> ' + command)
child = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True)
result = child.communicate()
cmd_result = CommandResult(child.returncode,
result[0].strip(), result[1].strip())
if cmd_result.exit_code:
logging.error("exit_code: '%d', stdout: '%s', stderr: '%s'" %
(cmd_result.exit_code, cmd_result.stdout, cmd_result.stderr))
sys.exit(1)
return cmd_result
class OpenStackK8sCluster(object):
def __init__(self, stack_name, identity_file):
self.stack_name = stack_name
self.identity_file = identity_file
self.servers = {}
self.vm_docker_images = set()
self.kubectl_version = 'unknown'
self.docker_version = 'unknown'
response = run_command_or_exit(OPENSTACK_GET_SERVER_IPS % stack_name,
"Get all stack servers and ip addressed using stack name").stdout
for line in response.split('\n'):
parts = line.split()
self.servers[parts[0].replace(stack_name+'-', "")] = parts[2]
def __str__(self):
desc = "Stack name: " + self.stack_name + '\n'
for key, value in sorted(self.servers.items()):
desc += " " + key + " : " + value + "\n"
return desc.strip()
def get_stack_name(self):
return self.stack_name
def get_identity_file(self):
return self.identity_file
def get_nfs_ip_address(self):
return self.servers['nfs']
def get_worker_nodes(self):
return [value for key, value in self.servers.items() if 'k8s-' in key.lower()]
def get_orchestrators(self):
return [value for key, value in self.servers.items() if 'orch-' in key.lower()]
def determine_docker_images_on_vms(self):
for node_ip in self.get_worker_nodes() + self.get_orchestrators():
command = SSH_CMD_TEMPLATE % (self.get_identity_file(), node_ip, DOCKER_INSPECT)
vm_inspect_results = run_command_or_exit(command, "Examine server and list docker images").stdout
for inspect_line in vm_inspect_results.split('\n'):
name_tag, name_digest = inspect_line.split(' ')
name, tag = name_tag.rsplit(':', 1)
digest = name_digest.split('sha256:')[1]
if local_registry:
name = name.replace(local_registry + "/", "")
self.vm_docker_images.add((name, tag, digest))
def get_docker_images_on_vms(self):
return self.vm_docker_images
def get_number_of_vm_docker_images(self):
return len(self.vm_docker_images)
def determine_kubectl_version(self):
command = SSH_CMD_TEMPLATE % (self.get_identity_file(), self.get_nfs_ip_address(), KUBECTL_VERSION)
self.kubectl_version = run_command_or_exit(command, "Examine nfs vm to determine kubectl version").stdout
def get_kubectl_version(self):
return self.kubectl_version
def determine_docker_version(self):
command = SSH_CMD_TEMPLATE % (self.get_identity_file(), self.get_worker_nodes()[0], DOCKER_VERSION)
self.docker_version = run_command_or_exit(command, "Examine worker node to determine docker version").stdout
def get_docker_version(self):
return self.docker_version
class OnapDeployment(object):
def __init__(self, openstack_stack):
self.stack = openstack_stack
self.raw = ""
self.pods = []
self.unique_images = set()
def dig(self):
command = SSH_CMD_TEMPLATE % (self.stack.get_identity_file(), self.stack.get_nfs_ip_address(),
KUBECTL_GET_ALL_POD_IMAGES_AND_SHAS)
self.raw = run_command_or_exit(command, "Use kubectl to retrieve all pods and pod images in K8S cluster").stdout
for row in self.raw.strip().split("\n"):
self.pods.append(Pod(row))
for pod in self.pods:
for image in pod.get_images():
self.unique_images.add(image)
def __str__(self):
desc = "Pods and docker images:\n"
for pod in sorted(self.pods):
desc += str(pod)
return desc.strip()
def get_number_of_pods(self):
return len(self.pods)
def get_docker_images(self):
return sorted(self.unique_images)
def get_number_of_unique_docker_images(self):
return len(self.unique_images)
def get_number_of_docker_images(self):
images = []
for pod in self.pods:
for image in pod.get_images():
images.append(image)
return len(images)
class Pod(object):
def __init__(self, data):
self.name, images = data.strip().split(":::")
self.shas_images = {}
for item in images.split(" "):
image_raw, sha_raw = item.split("___")
if local_registry:
image_raw = image_raw.replace(local_registry + "/", "")
if "sha256:" in images:
self.shas_images[sha_raw.split("sha256:")[1]] = image_raw
def get_images(self):
return self.shas_images.values()
def __cmp__(self, other):
return cmp(self.name, other.name)
def __str__(self):
desc = self.name + "\n"
for key, value in sorted(self.shas_images.items(), key=lambda x: x[1]):
desc += " " + value + ", " + key + "\n"
return desc
def main():
# Disable output buffering to receive the output instantly
sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0)
sys.stderr = os.fdopen(sys.stderr.fileno(), "w", 0)
help_desc = "Script to probe an ONAP k8s cluster on an OpenStack stack and report back information on the " \
"stack vms, all pods and corresponding docker image versions/digests and some of the tool versions " \
"utilised."
parser = argparse.ArgumentParser(description=help_desc)
parser.add_argument("-s", "--stack-name", dest="stack_name",
help="OpenStack stack name used by this ONAP deployment",
metavar="STACKNAME", required=True)
parser.add_argument("-i", "--identity_file", dest="identity_file",
help="OpenStack identity file to be used by ssh to access servers",
metavar="IDENTITY-FILE", required=True)
parser.add_argument("-r", "--registry", dest="registry",
help="Local registry used to serve docker images which should be " +
" stripped from any image names in the script output",
metavar="REGISTRY", required=False)
args = parser.parse_args()
if args.registry:
global local_registry
local_registry = args.registry
openstack_k8s = OpenStackK8sCluster(args.stack_name, args.identity_file)
openstack_k8s.determine_kubectl_version()
openstack_k8s.determine_docker_version()
openstack_k8s.determine_docker_images_on_vms()
onap_dep = OnapDeployment(openstack_k8s)
onap_dep.dig()
logging.info('\n%s\n' % openstack_k8s)
logging.info("number of pods: %d" % onap_dep.get_number_of_pods())
logging.info("number of docker images in pods: %d" % onap_dep.get_number_of_docker_images())
logging.info("number of unique docker images in pods: %d" % onap_dep.get_number_of_unique_docker_images())
logging.info("number of unique docker images on vms: %d" % openstack_k8s.get_number_of_vm_docker_images())
logging.info("docker version:\n%s" % openstack_k8s.get_docker_version())
logging.info("kubectl version:\n%s" % openstack_k8s.get_kubectl_version())
logging.info("\n%s\n" % onap_dep)
logging.info("<image-name>,<image-version>,<image-digest>")
for entry in sorted(openstack_k8s.get_docker_images_on_vms()):
logging.info('%s,%s,%s' % (entry[0], entry[1], entry[2]))
if __name__ == "__main__":
main()
|
