#!/usr/bin/env python """ This script attempts to probe an ONAP deployment in a Kubernetes cluster on an OpenStack stack and extract all pods and corresponding docker image versions along with some of the tool versions utilised. """ import os import sys import subprocess import argparse import logging OPENSTACK_GET_SERVER_IPS = 'openstack server list --name "^(%s-).*" -c Name -c Networks -f value --sort-column Name' SSH_CMD_TEMPLATE = 'ssh -o StrictHostKeychecking=no -i %s ubuntu@%s "sudo su -c \\"%s\\""' KUBECTL_GET_ALL_POD_IMAGES_AND_SHAS = 'kubectl get pods --all-namespaces -o=jsonpath=\'{range .items[*]}' \ '{\\\\\\"\\n\\\\\\"}{.metadata.name}{\\\\\\":::\\\\\\"}' \ '{range .status.containerStatuses[*]}{.image}{\\\\\\"___\\\\\\"}' \ '{.imageID}{\\\\\\" \\\\\\"}{end}{end}{\\\\\\"\\n\\\\\\"}\'' DOCKER_INSPECT = 'docker inspect --format=\'{{index .RepoTags 0}}{{\\\\\\" \\\\\\"}}{{index .RepoDigests 0}}\' ' \ '\$(docker images -q | uniq| tr \'\n\' \' \')' KUBECTL_VERSION = 'kubectl version' DOCKER_VERSION = 'docker --version' local_registry = "" logging.basicConfig(level=logging.DEBUG, format='%(message)s') file_log = logging.FileHandler(filename='onap-probe-report.txt', mode='w') file_log.setLevel(logging.INFO) formatter = logging.Formatter('%(message)s') file_log.setFormatter(formatter) logging.getLogger('').addHandler(file_log) class CommandResult(object): def __init__(self, exit_code, stdout, stderr): self.exit_code = exit_code self.stdout = stdout self.stderr = stderr def run_command_or_exit(command, message=""): if message: logging.debug(message) logging.debug('cmd> ' + command) child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) result = child.communicate() cmd_result = CommandResult(child.returncode, result[0].strip(), result[1].strip()) if cmd_result.exit_code: logging.error("exit_code: '%d', stdout: '%s', stderr: '%s'" % (cmd_result.exit_code, cmd_result.stdout, cmd_result.stderr)) sys.exit(1) return cmd_result class OpenStackK8sCluster(object): def __init__(self, stack_name, identity_file): self.stack_name = stack_name self.identity_file = identity_file self.servers = {} self.vm_docker_images = set() self.kubectl_version = 'unknown' self.docker_version = 'unknown' response = run_command_or_exit(OPENSTACK_GET_SERVER_IPS % stack_name, "Get all stack servers and ip addressed using stack name").stdout for line in response.split('\n'): parts = line.split() self.servers[parts[0].replace(stack_name+'-', "")] = parts[2] def __str__(self): desc = "Stack name: " + self.stack_name + '\n' for key, value in sorted(self.servers.items()): desc += " " + key + " : " + value + "\n" return desc.strip() def get_stack_name(self): return self.stack_name def get_identity_file(self): return self.identity_file def get_nfs_ip_address(self): return self.servers['nfs'] def get_worker_nodes(self): return [value for key, value in self.servers.items() if 'k8s-' in key.lower()] def get_orchestrators(self): return [value for key, value in self.servers.items() if 'orch-' in key.lower()] def determine_docker_images_on_vms(self): for node_ip in self.get_worker_nodes() + self.get_orchestrators(): command = SSH_CMD_TEMPLATE % (self.get_identity_file(), node_ip, DOCKER_INSPECT) vm_inspect_results = run_command_or_exit(command, "Examine server and list docker images").stdout for inspect_line in vm_inspect_results.split('\n'): name_tag, name_digest = inspect_line.split(' ') name, tag = name_tag.rsplit(':', 1) digest = name_digest.split('sha256:')[1] if local_registry: name = name.replace(local_registry + "/", "") self.vm_docker_images.add((name, tag, digest)) def get_docker_images_on_vms(self): return self.vm_docker_images def get_number_of_vm_docker_images(self): return len(self.vm_docker_images) def determine_kubectl_version(self): command = SSH_CMD_TEMPLATE % (self.get_identity_file(), self.get_nfs_ip_address(), KUBECTL_VERSION) self.kubectl_version = run_command_or_exit(command, "Examine nfs vm to determine kubectl version").stdout def get_kubectl_version(self): return self.kubectl_version def determine_docker_version(self): command = SSH_CMD_TEMPLATE % (self.get_identity_file(), self.get_worker_nodes()[0], DOCKER_VERSION) self.docker_version = run_command_or_exit(command, "Examine worker node to determine docker version").stdout def get_docker_version(self): return self.docker_version class OnapDeployment(object): def __init__(self, openstack_stack): self.stack = openstack_stack self.raw = "" self.pods = [] self.unique_images = set() def dig(self): command = SSH_CMD_TEMPLATE % (self.stack.get_identity_file(), self.stack.get_nfs_ip_address(), KUBECTL_GET_ALL_POD_IMAGES_AND_SHAS) self.raw = run_command_or_exit(command, "Use kubectl to retrieve all pods and pod images in K8S cluster").stdout for row in self.raw.strip().split("\n"): self.pods.append(Pod(row)) for pod in self.pods: for image in pod.get_images(): self.unique_images.add(image) def __str__(self): desc = "Pods and docker images:\n" for pod in sorted(self.pods): desc += str(pod) return desc.strip() def get_number_of_pods(self): return len(self.pods) def get_docker_images(self): return sorted(self.unique_images) def get_number_of_unique_docker_images(self): return len(self.unique_images) def get_number_of_docker_images(self): images = [] for pod in self.pods: for image in pod.get_images(): images.append(image) return len(images) class Pod(object): def __init__(self, data): self.name, images = data.strip().split(":::") self.shas_images = {} for item in images.split(" "): image_raw, sha_raw = item.split("___") if local_registry: image_raw = image_raw.replace(local_registry + "/", "") if "sha256:" in images: self.shas_images[sha_raw.split("sha256:")[1]] = image_raw def get_images(self): return self.shas_images.values() def __cmp__(self, other): return cmp(self.name, other.name) # pylint: disable=E0602 def __str__(self): desc = self.name + "\n" for key, value in sorted(self.shas_images.items(), key=lambda x: x[1]): desc += " " + value + ", " + key + "\n" return desc def main(): # Disable output buffering to receive the output instantly sys.stdout = os.fdopen(sys.stdout.fileno(), "w", 0) sys.stderr = os.fdopen(sys.stderr.fileno(), "w", 0) help_desc = "Script to probe an ONAP k8s cluster on an OpenStack stack and report back information on the " \ "stack vms, all pods and corresponding docker image versions/digests and some of the tool versions " \ "utilised." parser = argparse.ArgumentParser(description=help_desc) parser.add_argument("-s", "--stack-name", dest="stack_name", help="OpenStack stack name used by this ONAP deployment", metavar="STACKNAME", required=True) parser.add_argument("-i", "--identity_file", dest="identity_file", help="OpenStack identity file to be used by ssh to access servers", metavar="IDENTITY-FILE", required=True) parser.add_argument("-r", "--registry", dest="registry", help="Local registry used to serve docker images which should be " + " stripped from any image names in the script output", metavar="REGISTRY", required=False) args = parser.parse_args() if args.registry: global local_registry local_registry = args.registry openstack_k8s = OpenStackK8sCluster(args.stack_name, args.identity_file) openstack_k8s.determine_kubectl_version() openstack_k8s.determine_docker_version() openstack_k8s.determine_docker_images_on_vms() onap_dep = OnapDeployment(openstack_k8s) onap_dep.dig() logging.info('\n%s\n' % openstack_k8s) logging.info("number of pods: %d" % onap_dep.get_number_of_pods()) logging.info("number of docker images in pods: %d" % onap_dep.get_number_of_docker_images()) logging.info("number of unique docker images in pods: %d" % onap_dep.get_number_of_unique_docker_images()) logging.info("number of unique docker images on vms: %d" % openstack_k8s.get_number_of_vm_docker_images()) logging.info("docker version:\n%s" % openstack_k8s.get_docker_version()) logging.info("kubectl version:\n%s" % openstack_k8s.get_kubectl_version()) logging.info("\n%s\n" % onap_dep) logging.info(",,") for entry in sorted(openstack_k8s.get_docker_images_on_vms()): logging.info('%s,%s,%s' % (entry[0], entry[1], entry[2])) if __name__ == "__main__": main()