From 16fe29ac226610f79c8da1f691437ec7fe6e79c4 Mon Sep 17 00:00:00 2001 From: "k.kedron" Date: Fri, 5 Jun 2020 14:51:01 +0200 Subject: Improvement sdc-BE-init python scripts - Implemented retries when request fail - Moved configuration variables to the Properties file - Extended sdcBeProxy - Implemented script to run import/update (should fix deployment glitch) - Updated the import_Normatives recipes to use new script Issue-ID: SDC-2784 Signed-off-by: Krystian Kedron Change-Id: I83fab898783ad8d3b3d532af43d75bc54d033c33 --- .../recipes/3_import_Normatives.rb | 28 +++--------- .../src/main/resources/scripts/sdcBePy/__init__.py | 3 ++ .../resources/scripts/sdcBePy/common/errors.py | 7 +++ .../scripts/sdcBePy/common/healthCheck.py | 11 ++--- .../resources/scripts/sdcBePy/common/logger.py | 2 +- .../scripts/sdcBePy/common/normative/main.py | 50 +++++++++++++++++++--- .../sdcBePy/common/normative/toscaElements.py | 15 ++++--- .../scripts/sdcBePy/common/normative/toscaTypes.py | 42 ++++++++++-------- .../resources/scripts/sdcBePy/common/properties.py | 47 ++++++++++++++++++++ .../resources/scripts/sdcBePy/common/sdcBeProxy.py | 20 ++++++++- .../resources/scripts/sdcBePy/consumers/run.py | 5 ++- .../scripts/sdcBePy/tosca/data/beConfig.json | 5 ++- .../resources/scripts/sdcBePy/tosca/imports/run.py | 29 +++++-------- .../main/resources/scripts/sdcBePy/tosca/main.py | 22 ++++++++++ .../main/resources/scripts/sdcBePy/tosca/run.py | 29 +++++++++++++ .../resources/scripts/sdcBePy/tosca/upgrade/run.py | 29 +++++-------- .../main/resources/scripts/sdcBePy/users/run.py | 7 ++- catalog-be/src/main/resources/scripts/setup.py | 7 +-- 18 files changed, 253 insertions(+), 105 deletions(-) create mode 100644 catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py create mode 100644 catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py create mode 100644 catalog-be/src/main/resources/scripts/sdcBePy/tosca/run.py diff --git a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb index 79f880b146..e9e44c0df9 100644 --- a/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb +++ b/catalog-be/sdc-backend-init/chef-repo/cookbooks/sdc-catalog-be-setup/recipes/3_import_Normatives.rb @@ -22,29 +22,13 @@ bash "executing-import_Normatives" do tar -xvf normatives.tar.gz # executing the normatives - # add --debug to the importNormativeAll.py arguments to enable debug - - check_normative="/tmp/check_normative.out" - status_code=$(curl -k -s -o ${check_normative} -w "%{http_code}\\n" -X GET -H 'Content-Type: application/json;charset=UTF-8' -H 'USER_ID: jh0003' -H 'X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090' "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen") - if [ "$status_code" != 200 ] ; then - exit "$status_code" - fi - - #curl -s -X GET -H "Content-Type: application/json;charset=UTF-8" -H "USER_ID: jh0003" -H "X-ECOMP-RequestID: cbe744a0-037b-458f-aab5-df6e543c4090" "#{protocol}://#{be_ip}:#{be_port}/sdc2/rest/v1/screen" > ${check_normative} - - resources_len=`cat ${check_normative}| jq '.["resources"]|length'` - mkdir -p /var/lib/jetty/logs + # add --debug to the sdcinit command to enable debug + + cd /var/tmp/normatives/import/tosca + sdcinit #{param} > /var/lib/jetty/logs/init.log + rc=$? + if [[ $rc != 0 ]]; then exit $rc; fi - cd /var/tmp/normatives/import/tosca/ - if [ $resources_len -eq 0 ] ; then - sdcimportall #{param} > /var/lib/jetty/logs/importNormativeAll.log - rc=$? - if [[ $rc != 0 ]]; then exit $rc; fi - else - sdcupgradeall #{param} > /var/lib/jetty/logs/upgradeNormative.log - rc=$? - if [[ $rc != 0 ]]; then exit $rc; fi - fi EOH returns [0] end diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/__init__.py b/catalog-be/src/main/resources/scripts/sdcBePy/__init__.py index e69de29bb2..832967ab20 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/__init__.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/__init__.py @@ -0,0 +1,3 @@ +from sdcBePy.common.properties import Properties + +properties = Properties() diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py new file mode 100644 index 0000000000..42d699f98f --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/errors.py @@ -0,0 +1,7 @@ + +class ResourceCreationError(Exception): + + def __init__(self, message, error_code, resource_name=None): + self.message = message + self.error_code = error_code + self.resource_name = resource_name diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py index 7d8558d644..c99db5b434 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/healthCheck.py @@ -4,14 +4,13 @@ import time from argparse import ArgumentParser from datetime import datetime +from sdcBePy import properties from sdcBePy.common.bColors import BColors +from sdcBePy.common.properties import init_properties from sdcBePy.common.sdcBeProxy import SdcBeProxy colors = BColors() -RETRY_TIME = 10 -RETRY_ATTEMPTS = 10 - def check_backend(sdc_be_proxy=None, reply_append_count=1, be_host=None, be_port=None, scheme=None, debug=False): if sdc_be_proxy is None: @@ -24,13 +23,14 @@ def check_backend(sdc_be_proxy=None, reply_append_count=1, be_host=None, be_port else: print('[WARRING]: ' + datetime.now().strftime('%Y/%m/%d %H:%M:%S') + colors.FAIL + ' Backend not responding, try #' + str(i) + colors.END_C) - time.sleep(RETRY_TIME) + time.sleep(properties.retry_time) return False def run(be_host, be_port, protocol): - if not check_backend(reply_append_count=RETRY_ATTEMPTS, be_host=be_host, be_port=be_port, scheme=protocol): + if not check_backend(reply_append_count=properties.retry_attempts, be_host=be_host, + be_port=be_port, scheme=protocol): print('[ERROR]: ' + time.strftime('%Y/%m/%d %H:%M:%S') + colors.FAIL + ' Backend is DOWN :-(' + colors.END_C) sys.exit() @@ -44,6 +44,7 @@ def get_args(): args = parser.parse_args() + init_properties(10, 10) return [args.ip, args.port, 'https' if args.https else 'http'] diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py index e2e434fcff..17ef9afafb 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py @@ -16,7 +16,7 @@ def log(desc, arg): print(desc, arg) -def error_and_exit(error_code, error_desc): +def print_and_exit(error_code, error_desc): if error_code > 0: print("status={0}. {1}".format(error_code, '' if not error_desc else error_desc)) else: diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py index a30d46d4d5..ad8338cee4 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py @@ -1,16 +1,56 @@ +import time +from datetime import datetime + +from sdcBePy.common.bColors import BColors +from sdcBePy.common.logger import print_and_exit from sdcBePy.common.normative.toscaElements import process_and_create_normative_element from sdcBePy.common.normative.toscaTypes import process_and_create_normative_types +from sdcBePy.common.errors import ResourceCreationError +from sdcBePy import properties + +colors = BColors() def process_element_list(normative_elements_list, sdc_be_proxy): + attempt = 0 for normative_element in normative_elements_list: - process_and_create_normative_element(normative_element, - sdc_be_proxy=sdc_be_proxy) + while True: + attempt += 1 + try: + process_and_create_normative_element(normative_element, + sdc_be_proxy=sdc_be_proxy) + break + except ResourceCreationError as e: + _check_and_retry(attempt, e.error_code, e.message) + except Exception as e: + _check_and_retry(attempt, 1, str(e)) def process_type_list(normative_type_list, sdc_be_proxy, update_version): + attempt = 0 for normative_type in normative_type_list: - process_and_create_normative_types(normative_type, - sdc_be_proxy=sdc_be_proxy, - update_version=update_version) + while True: + attempt += 1 + try: + process_and_create_normative_types(normative_type, + sdc_be_proxy=sdc_be_proxy, + update_version=update_version) + break + except ResourceCreationError as e: + _check_and_retry(attempt, e.error_code, e.message) + normative_type.normative_types_list = _reduce(normative_type.normative_types_list, e.resource_name) + except Exception as e: + _check_and_retry(attempt, 1, str(e)) + + +def _check_and_retry(attempt, code, message): + if attempt == properties.retry_attempts + 1: + print_and_exit(code, message) + + print(colors.FAIL + '[WARRING]: ' + datetime.now().strftime('%Y/%m/%d %H:%M:%S') + + ' ' + message + ", try again: #" + str(attempt) + colors.END_C) + time.sleep(properties.retry_time) + +def _reduce(_list, element): + return _list[_list.index(element)::] diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py index 1d4e734351..5cdca0a095 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py @@ -3,8 +3,9 @@ import zipfile import pycurl -from sdcBePy.common.logger import debug, log, print_name_and_return_code, error_and_exit +from sdcBePy.common.logger import debug, log, print_name_and_return_code, print_and_exit from sdcBePy.common.sdcBeProxy import SdcBeProxy +from sdcBePy.common.errors import ResourceCreationError def process_and_create_normative_element(normative_element, @@ -49,9 +50,11 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name, http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data) if http_res is not None: debug("http response =", http_res) - debug("response buffer", str(sdc_be_proxy.con.buffer.getvalue(), "UTF-8")) + + response = sdc_be_proxy.get_response_from_buffer() + debug("response buffer", response) # c.close() - return element_name, http_res, sdc_be_proxy.con.buffer.getvalue() + return element_name, http_res, response except Exception as inst: print("ERROR=" + str(inst)) @@ -99,9 +102,9 @@ def print_and_check_result(result, exit_on_success): if result is not None: print_name_and_return_code(result[0], result[1]) if result[1] is None or result[1] not in [200, 201, 409]: - error_and_exit(1, "Failed to create the normatives elements!!") + raise ResourceCreationError("Failed to create the normatives elements!!", 1) else: if exit_on_success is True: - error_and_exit(0, "All normatives elements created successfully!!") + print_and_exit(0, "All normatives elements created successfully!!") else: - error_and_exit(1, "Results is None!!") + raise ResourceCreationError("Results is None!", 1) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py index 48b20ccb1e..5d64f448d3 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py @@ -3,7 +3,8 @@ import zipfile import pycurl -from sdcBePy.common.logger import print_name_and_return_code, error_and_exit, log, debug +from sdcBePy.common.errors import ResourceCreationError +from sdcBePy.common.logger import print_name_and_return_code, print_and_exit, log, debug from sdcBePy.common.sdcBeProxy import SdcBeProxy @@ -32,25 +33,27 @@ def print_and_check_results(results, update_version, exit_on_success=False): print("----------------------------------------") check_results_and_exit(results, update_version, exit_on_success) else: - error_and_exit(1, "results is none -> error occurred!!") + raise ResourceCreationError("Results is none -> error occurred!!", 1) def check_results_and_exit(results, update_version, exit_on_success): if not _results_ok(results, _get_response_code(update_version)): - error_and_exit(1, "Failed to create the normatives types !!") + raise ResourceCreationError("Failed to create the normatives types !!", 1) else: if exit_on_success: - error_and_exit(0, "All normatives types created successfully!!") + print_and_exit(0, "All normatives types created successfully!!") def _create_normatives_type(file_dir, sdc_be_proxy, types, update_version): results = [] response_codes = _get_response_code(update_version) - for normativeType in types: - result = _send_request(sdc_be_proxy, file_dir, normativeType, update_version) + for normative_type in types: + result = _send_request(sdc_be_proxy, file_dir, normative_type, update_version) results.append(result) if result[1] is None or result[1] not in response_codes: - print("Failed creating normative type " + normativeType + ". " + str(result[1])) + raise ResourceCreationError("Failed creating normative type " + normative_type + ". " + str(result[1]), + 1, + normative_type) return results @@ -67,12 +70,13 @@ def _send_request(sdc_be_proxy, file_dir, element_name, update_version): send = _create_send_body(file_dir, element_name) debug(send) - httpRes = sdc_be_proxy.post_file(url, send) - if httpRes is not None: - debug("http response=", httpRes) - debug(sdc_be_proxy.con.buffer.getvalue()) + http_res = sdc_be_proxy.post_file(url, send) + if http_res is not None: + debug("http response=", http_res) - return element_name, httpRes, sdc_be_proxy.con.buffer.getvalue() + response = sdc_be_proxy.get_response_from_buffer() + debug(response) + return element_name, http_res, response except Exception as inst: print("ERROR=" + str(inst)) @@ -90,15 +94,15 @@ def _create_send_body(file_dir, element_name): debug(path) current_json_file = file_dir + element_name + "/" + element_name + ".json" - jsonFile = open(current_json_file) + json_file = open(current_json_file) debug("before load json") - json_data = json.load(jsonFile, strict=False) + json_data = json.load(json_file, strict=False) debug(json_data) - jsonAsStr = json.dumps(json_data) + json_as_str = json.dumps(json_data) - return [('resourceMetadata', jsonAsStr), ('resourceZip', (pycurl.FORM_FILE, path))] + return [('resourceMetadata', json_as_str), ('resourceZip', (pycurl.FORM_FILE, path))] def _results_ok(results, response_codes): @@ -110,11 +114,11 @@ def _results_ok(results, response_codes): def _get_response_code(update_version): - responseCodes = [200, 201] + response_codes = [200, 201] if update_version is False: - responseCodes.append(409) + response_codes.append(409) - return responseCodes + return response_codes def _boolean_to_string(boolean_value): diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py new file mode 100644 index 0000000000..8426f34464 --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/properties.py @@ -0,0 +1,47 @@ +class Properties: + + def __init__(self, retry_time=0, + retry_attempts=0, resource_len=0): + self.retry_time = retry_time + self.retry_attempts = retry_attempts + self.resource_len = resource_len + + @property + def retry_time(self): + return self._retry_time + + @retry_time.setter + def retry_time(self, value): + self._validate(value) + self._retry_time = value + + @property + def retry_attempts(self): + return self._retry_attempts + + @retry_attempts.setter + def retry_attempts(self, value): + self._validate(value) + self._retry_attempts = value + + @property + def resource_len(self): + return self._resource_len + + @resource_len.setter + def resource_len(self, value): + self._validate(value) + self._resource_len = value + + @staticmethod + def _validate(value): + if value < 0: + raise ValueError("Properties below 0 is not possible") + + +def init_properties(retry_time, retry_attempts, resource_len=0): + from sdcBePy import properties + + properties.retry_time = retry_time + properties.retry_attempts = retry_attempts + properties.resource_len = resource_len diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py index 6fea657fab..d9aa260b80 100755 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py @@ -12,6 +12,9 @@ def get_url(ip, port, protocol): class SdcBeProxy: + BODY_SEPARATOR = "\r\n\r\n" + CHARTSET = 'UTF-8' + def __init__(self, be_ip, be_port, scheme, user_id="jh0003", debug=False, connector=None): if not check_arguments_not_none(be_ip, be_port, scheme, user_id): @@ -45,9 +48,20 @@ class SdcBeProxy: 'consumerPassword': password })) + def get_normatives(self): + return self.con.get("/sdc2/rest/v1/screen", with_buffer=True) + def post_file(self, path, multi_part_form_data): return self.con.post_file(path, multi_part_form_data) + def get_response_from_buffer(self): + value = self.con.buffer.getvalue() + self.con.buffer.truncate(0) + self.con.buffer.seek(0) + + response = value.decode(self.CHARTSET).split(self.BODY_SEPARATOR) + return response[1] if len(response) == 2 else response[0] + class CurlConnector: CONTENT_TYPE_HEADER = "Content-Type: application/json" @@ -71,13 +85,17 @@ class CurlConnector: self.url = url self._check_schema(scheme) - def get(self, path): + def get(self, path, buffer=None, with_buffer=False): try: self.c.setopt(pycurl.URL, self.url + path) self.c.setopt(pycurl.HTTPHEADER, [self.user_header, CurlConnector.CONTENT_TYPE_HEADER, CurlConnector.ACCEPT_HEADER]) + if with_buffer: + write = self.buffer.write if not buffer else buffer.write + self.c.setopt(pycurl.WRITEFUNCTION, write) + self.c.perform() return self.c.getinfo(pycurl.RESPONSE_CODE) except pycurl.error: diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/consumers/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/consumers/run.py index 59a0b610d8..1fb766227b 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/consumers/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/consumers/run.py @@ -1,6 +1,7 @@ import time -from sdcBePy.common.healthCheck import check_backend, RETRY_ATTEMPTS, get_args +from sdcBePy import properties +from sdcBePy.common.healthCheck import check_backend, get_args from sdcBePy.common.sdcBeProxy import SdcBeProxy from sdcBePy.consumers.models.consumerCandidateList import get_consumers from sdcBePy.users.run import colors @@ -8,7 +9,7 @@ from sdcBePy.users.run import colors def be_consumers_init(be_ip, be_port, protocol, consumer_candidate_list): sdc_be_proxy = SdcBeProxy(be_ip, be_port, protocol) - if check_backend(sdc_be_proxy, RETRY_ATTEMPTS): + if check_backend(sdc_be_proxy, properties.retry_attempts): for consumer in consumer_candidate_list: if sdc_be_proxy.check_user(consumer.consumer_name) != 200: result = sdc_be_proxy.create_consumer(*consumer.get_parameters()) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/data/beConfig.json b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/data/beConfig.json index f8c740b94f..556e561153 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/data/beConfig.json +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/data/beConfig.json @@ -1,5 +1,8 @@ { "beHost": "localhost", "bePort": "8080", - "adminUser": "jh0003" + "adminUser": "jh0003", + "resourceLen": 63, + "retryTime": 10, + "retryAttempt": 10 } \ No newline at end of file diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py index 4d4eeaff6e..9ac820071a 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py @@ -1,32 +1,18 @@ #!/usr/bin/env python3 import os -import sys import sdcBePy.common.logger as logger from sdcBePy.common.normative.main import process_element_list, process_type_list -from sdcBePy.common.sdcBeProxy import SdcBeProxy -from sdcBePy.tosca.main import get_args, usage +from sdcBePy.tosca.main import parse_and_create_proxy from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \ get_normative_element_with_metadata_list from sdcBePy.tosca.models.normativeTypesList import get_normative_type_candidate_list -def main(): - scheme, be_host, be_port, admin_user, update_version, debug = get_args() - - if debug is False: - print('Disabling debug mode') - logger.debugFlag = debug - - try: - sdc_be_proxy = SdcBeProxy(be_host, be_port, scheme, admin_user, debug) - except AttributeError: - usage() - sys.exit(3) - +def main(sdc_be_proxy, update_version): # use to run script form this dir (not like the command) - # base_file_location = os.getcwd() + "/../../../../import/tosca/" + # base_file_location = os.getcwd() + "/../../../import/tosca/" base_file_location = os.getcwd() + os.path.sep logger.debug("working directory =" + base_file_location) @@ -35,8 +21,13 @@ def main(): process_element_list(get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy) logger.log("Script end ->", "All normatives imported successfully!") - logger.error_and_exit(0, None) + logger.print_and_exit(0, None) + + +def run(): + sdc_be_proxy, update_version = parse_and_create_proxy() + main(sdc_be_proxy, update_version) if __name__ == "__main__": - main() + run() diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/main.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/main.py index 2535ba6d77..565ce7efdb 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/main.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/main.py @@ -3,6 +3,10 @@ import os import sys from argparse import ArgumentParser +from sdcBePy.common import logger +from sdcBePy.common.properties import init_properties +from sdcBePy.common.sdcBeProxy import SdcBeProxy + def usage(): print(sys.argv[0], @@ -53,4 +57,22 @@ def get_args(): print('scheme =', scheme, ',be host =', be_host, ', be port =', be_port, ', user =', admin_user, ', debug =', debug, ', update_version =', update_version) + init_properties(defaults["retryTime"], defaults["retryAttempt"], defaults["resourceLen"]) return scheme, be_host, be_port, admin_user, update_version, debug + + +def parse_and_create_proxy(): + scheme, be_host, be_port, admin_user, update_version, debug = get_args() + + if debug is False: + print('Disabling debug mode') + logger.debugFlag = debug + + try: + sdc_be_proxy = SdcBeProxy(be_host, be_port, scheme, admin_user, debug=debug) + except AttributeError: + usage() + sys.exit(3) + + return sdc_be_proxy, update_version + diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/run.py new file mode 100644 index 0000000000..955acff930 --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/run.py @@ -0,0 +1,29 @@ +import json + +from sdcBePy import properties +from sdcBePy.common.logger import print_and_exit + +from sdcBePy.tosca.imports.run import main as import_main +from sdcBePy.tosca.main import parse_and_create_proxy +from sdcBePy.tosca.upgrade.run import main as upgrade_main + + +def run(): + sdc_be_proxy, update_version = parse_and_create_proxy() + + response = sdc_be_proxy.get_normatives() + + resources = [] + if response == 200: + resources = json.loads(sdc_be_proxy.get_response_from_buffer())["resources"] + else: + print_and_exit(response, "Can't get normatives!") + + if len(resources) < properties.resource_len: + import_main(sdc_be_proxy, update_version) + else: + upgrade_main(sdc_be_proxy) + + +if __name__ == '__main__': + run() diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py index 6d90a1c085..47acd05f5f 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py @@ -1,35 +1,21 @@ #!/usr/bin/env python3 import os -import sys from sdcBePy.common import logger -from sdcBePy.common.logger import error_and_exit +from sdcBePy.common.logger import print_and_exit from sdcBePy.common.normative.main import process_element_list, process_type_list -from sdcBePy.common.sdcBeProxy import SdcBeProxy -from sdcBePy.tosca.main import get_args, usage +from sdcBePy.tosca.main import parse_and_create_proxy from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \ get_normative_element_with_metadata_list from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \ get_nfv_onap_sol_to_update_list -def main(): - scheme, be_host, be_port, admin_user, _, debug = get_args() - +def main(sdc_be_proxy): update_version = True update_onap_version = False - if debug is False: - print('Disabling debug mode') - logger.debugFlag = debug - - try: - sdc_be_proxy = SdcBeProxy(be_host, be_port, scheme, admin_user, debug=debug) - except AttributeError: - usage() - sys.exit(3) - # use to run script form this dir (not like the command) # base_file_location = os.getcwd() + "/../../../../import/tosca/" base_file_location = os.getcwd() + "/" @@ -46,7 +32,7 @@ def main(): process_type_list(nfv_onap_sol_list, sdc_be_proxy, update_onap_version) logger.log("Updating end ->", "All normatives updated successfully!") - error_and_exit(0, None) + print_and_exit(0, None) def get_all_types(): @@ -55,5 +41,10 @@ def get_all_types(): path + "/../data/onapTypesToUpgrade.json"]) +def run(): + sdc_be_proxy, _ = parse_and_create_proxy() + main(sdc_be_proxy) + + if __name__ == "__main__": - main() + run() diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/users/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/users/run.py index b933afe7ca..1518c2f673 100755 --- a/catalog-be/src/main/resources/scripts/sdcBePy/users/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/users/run.py @@ -5,8 +5,10 @@ import os import time from argparse import ArgumentParser +from sdcBePy import properties from sdcBePy.common.bColors import BColors -from sdcBePy.common.healthCheck import check_backend, RETRY_ATTEMPTS +from sdcBePy.common.healthCheck import check_backend +from sdcBePy.common.properties import init_properties from sdcBePy.common.sdcBeProxy import SdcBeProxy colors = BColors() @@ -19,7 +21,7 @@ def load_users(conf_path): def be_user_init(be_ip, be_port, protocol, conf_path): sdc_be_proxy = SdcBeProxy(be_ip, be_port, protocol) - if check_backend(sdc_be_proxy, RETRY_ATTEMPTS): + if check_backend(sdc_be_proxy, properties.retry_attempts): users = load_users(conf_path) for user in users: if sdc_be_proxy.check_user(user['userId']) != 200: @@ -53,6 +55,7 @@ def get_args(): args = parser.parse_args() + init_properties(10, 10) return [args.ip, args.port, 'https' if args.https else 'http', args.conf] diff --git a/catalog-be/src/main/resources/scripts/setup.py b/catalog-be/src/main/resources/scripts/setup.py index 9df26a7aa6..da2cad2da3 100644 --- a/catalog-be/src/main/resources/scripts/setup.py +++ b/catalog-be/src/main/resources/scripts/setup.py @@ -14,7 +14,7 @@ setup( entry_points={ "console_scripts": [ "sdcuserinit=sdcBePy.users.run:main", - "sdcimportall=sdcBePy.tosca.imports.run:main", + "sdcimportall=sdcBePy.tosca.imports.run:run", "sdcimportdata=sdcBePy.tosca.imports.runNormativeElement:run_import_data", "sdcimportcapabilities=sdcBePy.tosca.imports.runNormativeElement:run_import_capabilities", "sdcimportrelationship=sdcBePy.tosca.imports.runNormativeElement:run_import_relationship", @@ -29,7 +29,7 @@ setup( "sdcimportsol=sdcBePy.tosca.imports.runNormativeType:run_import_sol", "sdcimportannotation=sdcBePy.tosca.imports.runNormativeType:run_import_annotation", "sdcimportgeneric=sdcBePy.tosca.imports.runGenericNormative:main", - "sdcupgradeall=sdcBePy.tosca.upgrade.run:main", + "sdcupgradeall=sdcBePy.tosca.upgrade.run:run", "sdcupgradenfv=sdcBePy.tosca.upgrade.runUpgradeNormative:run_upgrade_nfv", "sdcupgradeonap=sdcBePy.tosca.upgrade.runUpgradeNormative:run_upgrade_onap", "sdcupgradesol=sdcBePy.tosca.upgrade.runUpgradeNormative:run_upgrade_sol", @@ -37,7 +37,8 @@ setup( "sdcupgradeheat1707_3537=sdcBePy.tosca.upgrade.runUpgradeNormative:run_upgrade_heat1707_3537", "sdcupgradeheatversion=sdcBePy.tosca.upgrade.runUpgradeNormative:run_upgrade_heat_version", "sdccheckbackend=sdcBePy.common.healthCheck:main", - "sdcconsumerinit=sdcBePy.consumers.run:main" + "sdcconsumerinit=sdcBePy.consumers.run:main", + "sdcinit=sdcBePy.tosca.run:run" ] }, install_requires=['pycurl'] ) -- cgit 1.2.3-korg