diff options
Diffstat (limited to 'catalog-be/src/main/resources/scripts/sdcBePy/common')
8 files changed, 157 insertions, 37 deletions
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py new file mode 100644 index 0000000000..42d699f98f --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py @@ -0,0 +1,7 @@ + +class ResourceCreationError(Exception): + + def __init__(self, message, error_code, resource_name=None): + self.message = message + self.error_code = error_code + self.resource_name = resource_name diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py index 7d8558d644..c99db5b434 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py @@ -4,14 +4,13 @@ import time from argparse import ArgumentParser from datetime import datetime +from sdcBePy import properties from sdcBePy.common.bColors import BColors +from sdcBePy.common.properties import init_properties from sdcBePy.common.sdcBeProxy import SdcBeProxy colors = BColors() -RETRY_TIME = 10 -RETRY_ATTEMPTS = 10 - def check_backend(sdc_be_proxy=None, reply_append_count=1, be_host=None, be_port=None, scheme=None, debug=False): if sdc_be_proxy is None: @@ -24,13 +23,14 @@ def check_backend(sdc_be_proxy=None, reply_append_count=1, be_host=None, be_port else: print('[WARRING]: ' + datetime.now().strftime('%Y/%m/%d %H:%M:%S') + colors.FAIL + ' Backend not responding, try #' + str(i) + colors.END_C) - time.sleep(RETRY_TIME) + time.sleep(properties.retry_time) return False def run(be_host, be_port, protocol): - if not check_backend(reply_append_count=RETRY_ATTEMPTS, be_host=be_host, be_port=be_port, scheme=protocol): + if not check_backend(reply_append_count=properties.retry_attempts, be_host=be_host, + be_port=be_port, scheme=protocol): print('[ERROR]: ' + time.strftime('%Y/%m/%d %H:%M:%S') + colors.FAIL + ' Backend is DOWN :-(' + colors.END_C) sys.exit() @@ -44,6 +44,7 @@ def get_args(): args = parser.parse_args() + init_properties(10, 10) return [args.ip, args.port, 'https' if args.https else 'http'] diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py index e2e434fcff..17ef9afafb 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py @@ -16,7 +16,7 @@ def log(desc, arg): print(desc, arg) -def error_and_exit(error_code, error_desc): +def print_and_exit(error_code, error_desc): if error_code > 0: print("status={0}. {1}".format(error_code, '' if not error_desc else error_desc)) else: diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py index a30d46d4d5..ad8338cee4 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py @@ -1,16 +1,56 @@ +import time +from datetime import datetime + +from sdcBePy.common.bColors import BColors +from sdcBePy.common.logger import print_and_exit from sdcBePy.common.normative.toscaElements import process_and_create_normative_element from sdcBePy.common.normative.toscaTypes import process_and_create_normative_types +from sdcBePy.common.errors import ResourceCreationError +from sdcBePy import properties + +colors = BColors() def process_element_list(normative_elements_list, sdc_be_proxy): + attempt = 0 for normative_element in normative_elements_list: - process_and_create_normative_element(normative_element, - sdc_be_proxy=sdc_be_proxy) + while True: + attempt += 1 + try: + process_and_create_normative_element(normative_element, + sdc_be_proxy=sdc_be_proxy) + break + except ResourceCreationError as e: + _check_and_retry(attempt, e.error_code, e.message) + except Exception as e: + _check_and_retry(attempt, 1, str(e)) def process_type_list(normative_type_list, sdc_be_proxy, update_version): + attempt = 0 for normative_type in normative_type_list: - process_and_create_normative_types(normative_type, - sdc_be_proxy=sdc_be_proxy, - update_version=update_version) + while True: + attempt += 1 + try: + process_and_create_normative_types(normative_type, + sdc_be_proxy=sdc_be_proxy, + update_version=update_version) + break + except ResourceCreationError as e: + _check_and_retry(attempt, e.error_code, e.message) + normative_type.normative_types_list = _reduce(normative_type.normative_types_list, e.resource_name) + except Exception as e: + _check_and_retry(attempt, 1, str(e)) + + +def _check_and_retry(attempt, code, message): + if attempt == properties.retry_attempts + 1: + print_and_exit(code, message) + + print(colors.FAIL + '[WARRING]: ' + datetime.now().strftime('%Y/%m/%d %H:%M:%S') + + ' ' + message + ", try again: #" + str(attempt) + colors.END_C) + time.sleep(properties.retry_time) + +def _reduce(_list, element): + return _list[_list.index(element)::] diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py index 1d4e734351..5cdca0a095 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py @@ -3,8 +3,9 @@ import zipfile import pycurl -from sdcBePy.common.logger import debug, log, print_name_and_return_code, error_and_exit +from sdcBePy.common.logger import debug, log, print_name_and_return_code, print_and_exit from sdcBePy.common.sdcBeProxy import SdcBeProxy +from sdcBePy.common.errors import ResourceCreationError def process_and_create_normative_element(normative_element, @@ -49,9 +50,11 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name, http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data) if http_res is not None: debug("http response =", http_res) - debug("response buffer", str(sdc_be_proxy.con.buffer.getvalue(), "UTF-8")) + + response = sdc_be_proxy.get_response_from_buffer() + debug("response buffer", response) # c.close() - return element_name, http_res, sdc_be_proxy.con.buffer.getvalue() + return element_name, http_res, response except Exception as inst: print("ERROR=" + str(inst)) @@ -99,9 +102,9 @@ def print_and_check_result(result, exit_on_success): if result is not None: print_name_and_return_code(result[0], result[1]) if result[1] is None or result[1] not in [200, 201, 409]: - error_and_exit(1, "Failed to create the normatives elements!!") + raise ResourceCreationError("Failed to create the normatives elements!!", 1) else: if exit_on_success is True: - error_and_exit(0, "All normatives elements created successfully!!") + print_and_exit(0, "All normatives elements created successfully!!") else: - error_and_exit(1, "Results is None!!") + raise ResourceCreationError("Results is None!", 1) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py index 48b20ccb1e..5d64f448d3 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py @@ -3,7 +3,8 @@ import zipfile import pycurl -from sdcBePy.common.logger import print_name_and_return_code, error_and_exit, log, debug +from sdcBePy.common.errors import ResourceCreationError +from sdcBePy.common.logger import print_name_and_return_code, print_and_exit, log, debug from sdcBePy.common.sdcBeProxy import SdcBeProxy @@ -32,25 +33,27 @@ def print_and_check_results(results, update_version, exit_on_success=False): print("----------------------------------------") check_results_and_exit(results, update_version, exit_on_success) else: - error_and_exit(1, "results is none -> error occurred!!") + raise ResourceCreationError("Results is none -> error occurred!!", 1) def check_results_and_exit(results, update_version, exit_on_success): if not _results_ok(results, _get_response_code(update_version)): - error_and_exit(1, "Failed to create the normatives types !!") + raise ResourceCreationError("Failed to create the normatives types !!", 1) else: if exit_on_success: - error_and_exit(0, "All normatives types created successfully!!") + print_and_exit(0, "All normatives types created successfully!!") def _create_normatives_type(file_dir, sdc_be_proxy, types, update_version): results = [] response_codes = _get_response_code(update_version) - for normativeType in types: - result = _send_request(sdc_be_proxy, file_dir, normativeType, update_version) + for normative_type in types: + result = _send_request(sdc_be_proxy, file_dir, normative_type, update_version) results.append(result) if result[1] is None or result[1] not in response_codes: - print("Failed creating normative type " + normativeType + ". " + str(result[1])) + raise ResourceCreationError("Failed creating normative type " + normative_type + ". " + str(result[1]), + 1, + normative_type) return results @@ -67,12 +70,13 @@ def _send_request(sdc_be_proxy, file_dir, element_name, update_version): send = _create_send_body(file_dir, element_name) debug(send) - httpRes = sdc_be_proxy.post_file(url, send) - if httpRes is not None: - debug("http response=", httpRes) - debug(sdc_be_proxy.con.buffer.getvalue()) + http_res = sdc_be_proxy.post_file(url, send) + if http_res is not None: + debug("http response=", http_res) - return element_name, httpRes, sdc_be_proxy.con.buffer.getvalue() + response = sdc_be_proxy.get_response_from_buffer() + debug(response) + return element_name, http_res, response except Exception as inst: print("ERROR=" + str(inst)) @@ -90,15 +94,15 @@ def _create_send_body(file_dir, element_name): debug(path) current_json_file = file_dir + element_name + "/" + element_name + ".json" - jsonFile = open(current_json_file) + json_file = open(current_json_file) debug("before load json") - json_data = json.load(jsonFile, strict=False) + json_data = json.load(json_file, strict=False) debug(json_data) - jsonAsStr = json.dumps(json_data) + json_as_str = json.dumps(json_data) - return [('resourceMetadata', jsonAsStr), ('resourceZip', (pycurl.FORM_FILE, path))] + return [('resourceMetadata', json_as_str), ('resourceZip', (pycurl.FORM_FILE, path))] def _results_ok(results, response_codes): @@ -110,11 +114,11 @@ def _results_ok(results, response_codes): def _get_response_code(update_version): - responseCodes = [200, 201] + response_codes = [200, 201] if update_version is False: - responseCodes.append(409) + response_codes.append(409) - return responseCodes + return response_codes def _boolean_to_string(boolean_value): diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py new file mode 100644 index 0000000000..8426f34464 --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py @@ -0,0 +1,47 @@ +class Properties: + + def __init__(self, retry_time=0, + retry_attempts=0, resource_len=0): + self.retry_time = retry_time + self.retry_attempts = retry_attempts + self.resource_len = resource_len + + @property + def retry_time(self): + return self._retry_time + + @retry_time.setter + def retry_time(self, value): + self._validate(value) + self._retry_time = value + + @property + def retry_attempts(self): + return self._retry_attempts + + @retry_attempts.setter + def retry_attempts(self, value): + self._validate(value) + self._retry_attempts = value + + @property + def resource_len(self): + return self._resource_len + + @resource_len.setter + def resource_len(self, value): + self._validate(value) + self._resource_len = value + + @staticmethod + def _validate(value): + if value < 0: + raise ValueError("Properties below 0 is not possible") + + +def init_properties(retry_time, retry_attempts, resource_len=0): + from sdcBePy import properties + + properties.retry_time = retry_time + properties.retry_attempts = retry_attempts + properties.resource_len = resource_len diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py index 6fea657fab..d9aa260b80 100755 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py @@ -12,6 +12,9 @@ def get_url(ip, port, protocol): class SdcBeProxy: + BODY_SEPARATOR = "\r\n\r\n" + CHARTSET = 'UTF-8' + def __init__(self, be_ip, be_port, scheme, user_id="jh0003", debug=False, connector=None): if not check_arguments_not_none(be_ip, be_port, scheme, user_id): @@ -45,9 +48,20 @@ class SdcBeProxy: 'consumerPassword': password })) + def get_normatives(self): + return self.con.get("/sdc2/rest/v1/screen", with_buffer=True) + def post_file(self, path, multi_part_form_data): return self.con.post_file(path, multi_part_form_data) + def get_response_from_buffer(self): + value = self.con.buffer.getvalue() + self.con.buffer.truncate(0) + self.con.buffer.seek(0) + + response = value.decode(self.CHARTSET).split(self.BODY_SEPARATOR) + return response[1] if len(response) == 2 else response[0] + class CurlConnector: CONTENT_TYPE_HEADER = "Content-Type: application/json" @@ -71,13 +85,17 @@ class CurlConnector: self.url = url self._check_schema(scheme) - def get(self, path): + def get(self, path, buffer=None, with_buffer=False): try: self.c.setopt(pycurl.URL, self.url + path) self.c.setopt(pycurl.HTTPHEADER, [self.user_header, CurlConnector.CONTENT_TYPE_HEADER, CurlConnector.ACCEPT_HEADER]) + if with_buffer: + write = self.buffer.write if not buffer else buffer.write + self.c.setopt(pycurl.WRITEFUNCTION, write) + self.c.perform() return self.c.getinfo(pycurl.RESPONSE_CODE) except pycurl.error: |