diff options
Diffstat (limited to 'robotframework-onap/eteutils')
| -rw-r--r-- | robotframework-onap/eteutils/DNSUtils.py | 17 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/EteGatherDataListener.py | 126 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/HEATUtils.py | 87 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/HTTPUtils.py | 18 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/JSONUtils.py | 41 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/OSUtils.py | 14 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/OpenstackLibrary.py | 124 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/RequestsClientCert.py | 7 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/StringTemplater.py | 9 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/TemplatingEngine.py | 34 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/UUID.py | 15 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/__init__.py | 0 | ||||
| -rw-r--r-- | robotframework-onap/eteutils/csvLibrary.py | 16 |
13 files changed, 508 insertions, 0 deletions
diff --git a/robotframework-onap/eteutils/DNSUtils.py b/robotframework-onap/eteutils/DNSUtils.py new file mode 100644 index 0000000..65ae68b --- /dev/null +++ b/robotframework-onap/eteutils/DNSUtils.py @@ -0,0 +1,17 @@ +import dns.message +import dns.name +import dns.query + +class DNSUtils: + """ Utilities useful for DNS requests """ + + def dns_request(self, domain, ns): + """ return the ip address of the given domain name from the given nameserver """ + request = dns.message.make_query(domain, dns.rdatatype.A); + request.flags |= dns.flags.AD; + request.find_rrset(request.additional, dns.name.root, 65535, dns.rdatatype.OPT, create=True, force_unique=True) + response = dns.query.udp(request, ns) + + for answer in response.answer: + for item in answer.items: + return item
\ No newline at end of file diff --git a/robotframework-onap/eteutils/EteGatherDataListener.py b/robotframework-onap/eteutils/EteGatherDataListener.py new file mode 100644 index 0000000..79c02b4 --- /dev/null +++ b/robotframework-onap/eteutils/EteGatherDataListener.py @@ -0,0 +1,126 @@ +import os.path +import paramiko +import logging +from sys import stderr + +""" +EteGatherDataListener implements the ROBOT listener API version 2 and is +instantiated via the robot cammmand line option + + --listener EteGatherDataListener:<jobbumber>:<key filename> + +The purpose is to gather and preserve debugging data from each of the application +VMs when an ETE test fails. + +This listener counts the number of test +cases that have failed and, if > 0 at then end of the robot exection (close()), +will connect to each application vm and + +2. upload the gather_data.sh +2. execute gather_data.sh +3. Transfer the resulting zip file to the Robot reports folder + +This will enable the Jenkins job to retrieve the debug data along with the +Robot logs and reports and archive it with the failed job for later retreival. + +Note that the gather_data.sh depends upon the application providing +a /opt/gather_application_data.sh on their respective VMs for the zip file +to be created. +""" + + +class EteGatherDataListener(object): + ROBOT_LIBRARY_SCOPE = 'TEST SUITE' + ROBOT_LISTENER_API_VERSION = 2 + + APPLICATIONS = { + "aai" : "10.0.1.1", + "appc" : "10.0.2.1", + "sdc" : "10.0.3.1", + "dcae" : "10.0.4.1", + "mso" : "10.0.5.1", + "policy" : "10.0.6.1", + "sdnc" : "10.0.7.1", + "vid" : "10.0.8.1", + "portal" : "10.0.9.1", + "message_router" : "10.0.11.1", + "dcae_pstg00" : "10.0.4.101", + "dcae_coll00" : "10.0.4.102", + "dcae_cdap00" : "10.0.4.103", + "dcae_cdap01" : "10.0.4.104", + "dcae_cdap02" : "10.0.4.105" + } + + keyfile = "" + local_gather_data_sh = "" + + def __init__(self, job='10', keyfile='/share/config/key.pvt', shell="gather_data.sh"): + self.tests_passed = 0 + self.tests_failed = 0 + self.output_folder = '' + self.job = job + self.folder= '' + self.keyfile = keyfile + self.local_gather_data_sh = shell + print "EteGatherDataListener instantiated" + + def end_test(self, name, attrs): + if attrs['status'] == 'PASS': + self.tests_passed+=1 + else: + self.tests_failed+=1 + + def output_file(self, path): + if (self.folder != ''): + return + self.folder = os.path.dirname(path) + print(self.folder) + + def close(self): + print "EteGatherDataListener tests failed=" + str(self.tests_failed) + if (self.tests_failed > 0): + self.gather_debug_data() + + def gather_debug_data(self): + + for application in self.APPLICATIONS.keys(): + self.gather_application_data(application, self.APPLICATIONS.get(application)) + + def gather_application_data(self, application, ip): + extra = {"_threadid" : 1} + paramiko.util.log_to_file(self.folder + "/paramiko.log", level=0) + log = logging.getLogger("paramiko") + ssh = paramiko.SSHClient() + try: + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip,username="root", key_filename=self.keyfile) + except paramiko.SSHException: + log.error("Connection Failed to " + ip, extra=extra) + return + try: + gather_data_sh = "/tmp/gather_data.sh" + ftp = ssh.open_sftp() + ftp.put(self.local_gather_data_sh, gather_data_sh) + ftp.close() + stdin, stdout, stderr = ssh.exec_command("/bin/bash "+ gather_data_sh + " " + application + " " + self.job) + error = stderr.read() + if (error != ''): + log.info("stderr:" + error, extra=extra) + ssh.close() + return; + # No error? ASsume we have a file to download. + out = stdout.read() + log.info("stdout:" + out, extra=extra) + filename = application + "_" + self.job + ".tar.gz" + localzip = self.folder + "/" + filename + remotezip = "/tmp/gather_data/" + filename + ftp = ssh.open_sftp() + ftp.get(remotezip, localzip) + ftp.close() + stdin, stdout, stderr = ssh.exec_command("rm -rf " + remotezip); + ssh.close() + except paramiko.SSHException: + ssh.close() + return + + diff --git a/robotframework-onap/eteutils/HEATUtils.py b/robotframework-onap/eteutils/HEATUtils.py new file mode 100644 index 0000000..15c5689 --- /dev/null +++ b/robotframework-onap/eteutils/HEATUtils.py @@ -0,0 +1,87 @@ +import json +import yaml +import StringIO +import copy +from hashlib import md5 +from paramiko import RSAKey +from paramiko.ssh_exception import PasswordRequiredException + +class HEATUtils: + """ Utilities useful for constructing OpenStack HEAT requests """ + + def get_yaml(self, template_file): + """Template Yaml To Json reads a YAML Heat template file returns a JSON string that can be used included in an Openstack Add Stack Request""" + if isinstance(template_file, basestring): + fin = open(template_file, 'r') + yamlobj = yaml.load(fin) + return yamlobj + return None + + def template_yaml_to_json(self, template_file): + """Template Yaml To Json reads a YAML Heat template file returns a JSON string that can be used included in an Openstack Add Stack Request""" + if isinstance(template_file, basestring): + fin = open(template_file, 'r') + yamlobj = yaml.load(fin) + fin.close() + if 'heat_template_version' in yamlobj: + datetime = yamlobj['heat_template_version'] + yamlobj['heat_template_version'] = str(datetime) + fout = StringIO.StringIO() + json.dump(yamlobj, fout) + contents = fout.getvalue() + fout.close() + return contents + + def env_yaml_to_json(self, template_file): + """Env Yaml To JSon reads a YAML Heat env file and returns a JSON string that can be used included in an Openstack Add Stack Request""" + if isinstance(template_file, basestring): + fin = open(template_file, 'r') + yamlobj = yaml.load(fin) + fin.close() + if 'parameters' in yamlobj: + fout = StringIO.StringIO() + json.dump(yamlobj['parameters'], fout) + contents = fout.getvalue() + fout.close() + return contents + return None + + def stack_info_parse(self, stack_info): + """ returns a flattened version of the Openstack Find Stack results """ + d = {} + if isinstance(stack_info, dict): + s = stack_info['stack'] + p = s['parameters'] + d = copy.deepcopy(p) + d['id'] = s['id'] + d['name'] = s['stack_name'] + d['stack_status'] = s['stack_status'] + return d + + + def match_fingerprint(self, pvt_file, pw, fingerprint): + try: + sshKey = RSAKey.from_private_key_file(pvt_file, pw) + keybytes = md5(sshKey.asbytes()).hexdigest() + printableFingerprint = ':'.join(a+b for a,b in zip(keybytes[::2], keybytes[1::2])) + return printableFingerprint == fingerprint.__str__() + except PasswordRequiredException: + return False + + def match_private_key_file_to_keypair(self, files, keypair): + for keyfile in files: + if (self.match_fingerprint(keyfile, None, keypair['keypair']['fingerprint'])): + return keyfile + return None + + def get_openstack_server_ip(self, server, network_name="public", ipversion=4): + ipaddr = None + try: + versions = server['addresses'][network_name] + for version in versions: + if version['version'] == ipversion: + ipaddr = version['addr'] + break; + except ValueError: + return ipaddr + return ipaddr
\ No newline at end of file diff --git a/robotframework-onap/eteutils/HTTPUtils.py b/robotframework-onap/eteutils/HTTPUtils.py new file mode 100644 index 0000000..9324af7 --- /dev/null +++ b/robotframework-onap/eteutils/HTTPUtils.py @@ -0,0 +1,18 @@ +import urllib +import urllib3 +import urlparse + +class HTTPUtils: + """HTTPUtils is common resource for simple http helper keywords.""" + + def url_encode_string(self, barestring): + """URL Encode String takes in a string and converts into 'percent-encoded' string""" + return urllib.quote_plus(barestring) + + def disable_warnings(self): + """ Disable the cert warnings when creating sessions for A&AI API Calls """ + urllib3.disable_warnings() + + def url_parse(self, url): + """ Get pieces of the URL """ + return urlparse.urlparse(url)
\ No newline at end of file diff --git a/robotframework-onap/eteutils/JSONUtils.py b/robotframework-onap/eteutils/JSONUtils.py new file mode 100644 index 0000000..de5da6b --- /dev/null +++ b/robotframework-onap/eteutils/JSONUtils.py @@ -0,0 +1,41 @@ +import json + +from deepdiff import DeepDiff + +class JSONUtils: + """JSONUtils is common resource for simple json helper keywords.""" + + def json_equals(self, left, right): + """JSON Equals takes in two strings or json objects, converts them into json if needed and then compares them, returning if they are equal or not.""" + if isinstance(left, basestring): + left_json = json.loads(left); + else: + left_json = left; + if isinstance(right, basestring): + right_json = json.loads(right); + else: + right_json = right; + + ddiff = DeepDiff(left_json, right_json, ignore_order=True); + if ddiff == {}: + return True; + else: + return False; + + def make_list_into_dict(self, listOfDicts, key): + """ Converts a list of dicts that contains a field that has a unique key into a dict of dicts """ + d = {} + if isinstance(listOfDicts, list): + for thisDict in listOfDicts: + v = thisDict[key] + d[v] = thisDict + return d + + def find_element_in_array(self, searchedArray, key, value): + """ Takes in an array and a key value, it will return the items in the array that has a key and value that matches what you pass in """ + elements = []; + for item in searchedArray: + if key in item: + if item[key] == value: + elements.append(item); + return elements;
\ No newline at end of file diff --git a/robotframework-onap/eteutils/OSUtils.py b/robotframework-onap/eteutils/OSUtils.py new file mode 100644 index 0000000..78968f0 --- /dev/null +++ b/robotframework-onap/eteutils/OSUtils.py @@ -0,0 +1,14 @@ +from sys import platform + +class OSUtils: + """ Utilities useful for constructing OpenStack HEAT requests """ + + def get_normalized_os(self): + os = platform + if platform == "linux" or platform == "linux2": + os = 'linux64' + elif platform == "darwin": + os = 'mac64' + elif platform == "win32": + os = platform + return os diff --git a/robotframework-onap/eteutils/OpenstackLibrary.py b/robotframework-onap/eteutils/OpenstackLibrary.py new file mode 100644 index 0000000..adb12db --- /dev/null +++ b/robotframework-onap/eteutils/OpenstackLibrary.py @@ -0,0 +1,124 @@ +from robot.libraries.BuiltIn import BuiltIn +import robot.utils +import json + +class OpenstackLibrary: + """OpenstackLibrary manages the connection state and service catalog of an openstack instance.""" + + ROBOT_LIBRARY_SCOPE = 'Global' + + + def __init__(self): + self._cache = robot.utils.ConnectionCache('No connections created') + self.builtin = BuiltIn() + + def save_openstack_auth(self, alias, response,token, version='v2.0'): + """Save Openstack Auth takes in an openstack auth response and saves it to allow easy retrival of token and service catalog""" + self.builtin.log('Creating connection: %s' % alias, 'DEBUG') + jsonResponse = json.loads(response); + jsonResponse['auth_token'] = token + jsonResponse['keystone_api_version'] = version + self._cache.register(jsonResponse, alias=alias) + + def get_openstack_token(self, alias): + """Get Openstack auth token from the current alias""" + response = self._cache.switch(alias) + if isinstance(response, basestring): + jsonResponse = json.loads(response); + else: + jsonResponse = response; + if jsonResponse['keystone_api_version'] == 'v2.0': + return jsonResponse['access']['token']['id'] + else: + return jsonResponse['auth_token'] + + def get_openstack_catalog(self, alias): + """Get Openstack service catalog from the current alias""" + response = self._cache.switch(alias) + if isinstance(response, basestring): + jsonResponse = json.loads(response); + else: + jsonResponse = response; + if jsonResponse['keystone_api_version'] == 'v2.0': + return jsonResponse['access']['serviceCatalog'] + else: + return jsonResponse['token']['catalog'] + + + def get_current_openstack_tenant(self, alias): + """Get Openstack tenant from the current alias""" + response = self._cache.switch(alias) + if isinstance(response, basestring): + jsonResponse = json.loads(response); + else: + jsonResponse = response; + if jsonResponse['keystone_api_version'] == 'v2.0': + return jsonResponse['access']['token']['tenant'] + else: + return jsonResponse['token']['project'] + + def get_current_openstack_tenant_id(self, alias): + """Get Openstack tenant id from the current alias""" + tenant = self.get_current_openstack_tenant(alias); + return tenant['id'] + + def get_openstack_regions(self, alias): + """Get all Openstack regions from the current alias""" + response = self._cache.switch(alias) + if isinstance(response, basestring): + jsonResponse = json.loads(response); + else: + jsonResponse = response; + regions = []; + if jsonResponse['keystone_api_version'] == 'v2.0': + resp = jsonResponse['access']['serviceCatalog'] + else: + resp = jsonResponse['token']['catalog'] + for catalogEntry in resp: + listOfEndpoints = catalogEntry['endpoints']; + for endpoint in listOfEndpoints: + if 'region'in endpoint: + if endpoint['region'] not in regions: + regions.append(endpoint['region']) + return regions; + + def get_openstack_service_url(self, alias, servicetype, region = None, tenant_id = None): + """Get Openstack service catalog from the current alias""" + response = self._cache.switch(alias) + if isinstance(response, basestring): + jsonResponse = json.loads(response); + else: + jsonResponse = response; + endPoint = None; + if jsonResponse['keystone_api_version'] == 'v2.0': + resp = jsonResponse['access']['serviceCatalog'] + else: + resp = jsonResponse['token']['catalog'] + for catalogEntry in resp: + if self.__determine_match(catalogEntry['type'], servicetype): + listOfEndpoints = catalogEntry['endpoints']; + # filter out non matching regions if provided + listOfEndpoints[:] = [x for x in listOfEndpoints if self.__determine_match(x['region'], region)]; + # filter out non matching tenants if provided + # Only provide tenant id when authorizing without qualifying with tenant id + # WindRiver does not return the tenantId on the endpoint in this case. + if tenant_id is not None: + listOfEndpoints[:] = [y for y in listOfEndpoints if self.__determine_match(y['tenantId'], tenant_id)]; + if jsonResponse['keystone_api_version'] == 'v3': + listOfEndpoints[:] = [z for z in listOfEndpoints if self.__determine_match(z['interface'], 'public')]; + if len(listOfEndpoints) > 0: + if jsonResponse['keystone_api_version'] == 'v2.0': + endPoint = listOfEndpoints[0]['publicURL']; + else: + endPoint = listOfEndpoints[0]['url']; + if endPoint == None: + self.builtin.should_not_be_empty("", "Service Endpoint Url should not be empty") + return endPoint; + + def __determine_match(self, listItem, item): + if item is None: + return True; + elif listItem == item: + return True; + else: + return False;
\ No newline at end of file diff --git a/robotframework-onap/eteutils/RequestsClientCert.py b/robotframework-onap/eteutils/RequestsClientCert.py new file mode 100644 index 0000000..e1fd66f --- /dev/null +++ b/robotframework-onap/eteutils/RequestsClientCert.py @@ -0,0 +1,7 @@ + +class RequestsClientCert: + """RequestsClientCert allows adding a client cert to the Requests Robot Library.""" + + def add_client_cert(self, session, cert): + """Add Client Cert takes in a requests session object and a string path to the cert""" + session.cert = cert
\ No newline at end of file diff --git a/robotframework-onap/eteutils/StringTemplater.py b/robotframework-onap/eteutils/StringTemplater.py new file mode 100644 index 0000000..43d107e --- /dev/null +++ b/robotframework-onap/eteutils/StringTemplater.py @@ -0,0 +1,9 @@ +import json +from string import Template + +class StringTemplater: + """StringTemplater is common resource for templating with strings.""" + + def template_string(self, template, values): + """Template String takes in a string and its values and converts it using the string.Template class""" + return Template(template).substitute(values)
\ No newline at end of file diff --git a/robotframework-onap/eteutils/TemplatingEngine.py b/robotframework-onap/eteutils/TemplatingEngine.py new file mode 100644 index 0000000..0f579e7 --- /dev/null +++ b/robotframework-onap/eteutils/TemplatingEngine.py @@ -0,0 +1,34 @@ +# Copyright 2019 AT&T Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from jinja2 import Environment, FileSystemLoader, select_autoescape + + +class TemplatingEngine: + """TemplateImporter is common resource for templating with strings.""" + + jinja_env = None + + def __init__(self, templates_folder): + self.jinja_env = Environment( + loader=FileSystemLoader(templates_folder), + autoescape=select_autoescape(['html', 'xml']) + ) + + def apply_template(self, template_location, values): + """returns a string that is the jinja template in template_location filled in via the dictionary in values """ + print + template = self.jinja_env.get_template(template_location) + return template.render(values)
\ No newline at end of file diff --git a/robotframework-onap/eteutils/UUID.py b/robotframework-onap/eteutils/UUID.py new file mode 100644 index 0000000..35c26a7 --- /dev/null +++ b/robotframework-onap/eteutils/UUID.py @@ -0,0 +1,15 @@ +import uuid +import time +import datetime + +class UUID: + """UUID is a simple library that generates a uuid""" + + def generate_UUID(self): + """generate a uuid""" + return uuid.uuid4() + + def generate_MilliTimestamp_UUID(self): + """generate a millisecond timestamp uuid""" + then = datetime.datetime.now() + return int(time.mktime(then.timetuple())*1e3 + then.microsecond/1e3)
\ No newline at end of file diff --git a/robotframework-onap/eteutils/__init__.py b/robotframework-onap/eteutils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/robotframework-onap/eteutils/__init__.py diff --git a/robotframework-onap/eteutils/csvLibrary.py b/robotframework-onap/eteutils/csvLibrary.py new file mode 100644 index 0000000..b38b4a5 --- /dev/null +++ b/robotframework-onap/eteutils/csvLibrary.py @@ -0,0 +1,16 @@ +import csv +class csvLibrary(object): + + def read_csv_file(self, filename): + '''This creates a keyword named "Read CSV File" + + This keyword takes one argument, which is a path to a .csv file. It + returns a list of rows, with each row being a list of the data in + each column. + ''' + data = [] + with open(filename, 'rb') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + data.append(row) + return data |