From ddf9aaefc753b492fb72144d597a27df8080a4ab Mon Sep 17 00:00:00 2001 From: "andre.schmid" Date: Fri, 28 May 2021 19:10:30 +0100 Subject: Init ONAP model imports using the model API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creates a client for the model endpoint in the catalog init scripts. Introduces the directory structure to provide the models along its imports, separated by init/upgrade phase. Each model structure will be zipped and uploaded to the endpoint, based on the model directory name. Change-Id: I0392c1e6d3a29b30567b11016041a8e9cccbc745 Issue-ID: SDC-3615 Signed-off-by: André Schmid --- .../sdcBePy/common/normative/toscaElements.py | 1 + .../resources/scripts/sdcBePy/common/sdcBeProxy.py | 74 +++++++++++------- .../resources/scripts/sdcBePy/tosca/imports/run.py | 21 +++-- .../scripts/sdcBePy/tosca/models/model_client.py | 81 +++++++++++++++++++ .../sdcBePy/tosca/models/model_import_manager.py | 91 ++++++++++++++++++++++ .../resources/scripts/sdcBePy/tosca/upgrade/run.py | 11 +++ 6 files changed, 245 insertions(+), 34 deletions(-) create mode 100644 catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py create mode 100644 catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py (limited to 'catalog-be/src/main/resources/scripts') diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py index 5ef3173907..654a755863 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py @@ -47,6 +47,7 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name, multi_part_form_data = _create_multipart_form_data(element_form_name, type_file_name, with_metadata, element_name) + debug("http request url =", url_suffix) http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data) if http_res is not None: debug("http response =", http_res) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py index 169979ae73..f2bef53564 100755 --- a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py @@ -21,7 +21,7 @@ class SdcBeProxy: raise AttributeError("The be_host, be_port, scheme or admin_user are missing") url = get_url(be_ip, be_port, scheme) self.con = connector if connector \ - else CurlConnector(url, user_id, header, scheme=scheme, debug=debug) + else CurlConnector(url, user_id, header, protocol=scheme, debug=debug) def check_backend(self): return self.con.get('/sdc2/rest/v1/user/jh0003') @@ -52,8 +52,11 @@ class SdcBeProxy: def get_normatives(self): return self.con.get("/sdc2/rest/v1/screen", with_buffer=True) - def post_file(self, path, multi_part_form_data): - return self.con.post_file(path, multi_part_form_data) + def post_file(self, path, multi_part_form_data, buffer=None): + return self.con.post_file(path, multi_part_form_data, buffer) + + def put_file(self, path, multi_part_form_data, buffer=None): + return self.con.put_file(path, multi_part_form_data, buffer) def get_response_from_buffer(self): value = self.con.buffer.getvalue() @@ -68,17 +71,13 @@ class CurlConnector: CONTENT_TYPE_HEADER = "Content-Type: application/json" ACCEPT_HEADER = "Accept: application/json; charset=UTF-8" - def __init__(self, url, user_id_header, header, buffer=None, scheme="http", debug=False): - self.c = pycurl.Curl() - self.c.setopt(pycurl.HEADER, True) + def __init__(self, url, user_id_header, header, buffer=None, protocol="http", debug=False): + self.__debug = debug + self.__protocol = protocol + self.c = self.__build_default_curl() self.user_header = "USER_ID: " + user_id_header - - if not debug: - # disable printing not necessary logs in the terminal - self.c.setopt(pycurl.WRITEFUNCTION, lambda x: None) - else: - self.c.setopt(pycurl.VERBOSE, 1) + self.url = url if not buffer: self.buffer = BytesIO() @@ -88,9 +87,6 @@ class CurlConnector: else: self.basicauth_header = "Authorization: Basic " + header - self.url = url - self._check_schema(scheme) - def get(self, path, buffer=None, with_buffer=False): try: self.c.setopt(pycurl.URL, self.url + path) @@ -99,7 +95,6 @@ class CurlConnector: CurlConnector.ACCEPT_HEADER, self.basicauth_header]) - if with_buffer: write = self.buffer.write if not buffer else buffer.write self.c.setopt(pycurl.WRITEFUNCTION, write) @@ -115,9 +110,9 @@ class CurlConnector: self.c.setopt(pycurl.POST, 1) self.c.setopt(pycurl.HTTPHEADER, [self.user_header, - CurlConnector.CONTENT_TYPE_HEADER, - CurlConnector.ACCEPT_HEADER, - self.basicauth_header]) + CurlConnector.CONTENT_TYPE_HEADER, + CurlConnector.ACCEPT_HEADER, + self.basicauth_header]) self.c.setopt(pycurl.POSTFIELDS, data) @@ -132,9 +127,7 @@ class CurlConnector: try: self.c.setopt(pycurl.URL, self.url + path) self.c.setopt(pycurl.POST, 1) - self.c.setopt(pycurl.HTTPHEADER, [self.user_header, - self.basicauth_header]) - + self.c.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header]) self.c.setopt(pycurl.HTTPPOST, post_body) @@ -143,15 +136,40 @@ class CurlConnector: self.c.perform() self.c.setopt(pycurl.POST, 0) - return self.c.getinfo(pycurl.RESPONSE_CODE) - except pycurl.error: + except pycurl.error as ex: + print(ex) return 111 - def _check_schema(self, scheme): - if scheme == 'https': - self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.SSL_VERIFYHOST, 0) + def put_file(self, path, post_body, response_write_buffer=None): + curl = self.__build_default_curl() + curl.setopt(pycurl.URL, self.url + path) + curl.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header]) + curl.setopt(pycurl.CUSTOMREQUEST, "PUT") + + curl.setopt(pycurl.HTTPPOST, post_body) + + write = self.buffer.write if not response_write_buffer else response_write_buffer.write + curl.setopt(pycurl.WRITEFUNCTION, write) + + curl.perform() + response_code = curl.getinfo(pycurl.RESPONSE_CODE) + curl.close() + return response_code + + def __build_default_curl(self): + curl = pycurl.Curl() + if not self.__debug: + # disable printing not necessary logs in the terminal + curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) + else: + curl.setopt(pycurl.VERBOSE, 1) + + if self.__protocol == 'https': + curl.setopt(pycurl.SSL_VERIFYPEER, 0) + curl.setopt(pycurl.SSL_VERIFYHOST, 0) + curl.setopt(pycurl.HEADER, True) + return curl def __del__(self): self.c.close() diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py index 9ac820071a..067f110efc 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py @@ -1,13 +1,15 @@ #!/usr/bin/env python3 import os +from pathlib import Path import sdcBePy.common.logger as logger from sdcBePy.common.normative.main import process_element_list, process_type_list from sdcBePy.tosca.main import parse_and_create_proxy -from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \ - get_normative_element_with_metadata_list -from sdcBePy.tosca.models.normativeTypesList import get_normative_type_candidate_list +from sdcBePy.tosca.models import normativeElementsList +from sdcBePy.tosca.models import normativeTypesList +from sdcBePy.tosca.models.model_client import ModelClient +from sdcBePy.tosca.models.model_import_manager import ModelImportManager def main(sdc_be_proxy, update_version): @@ -16,9 +18,16 @@ def main(sdc_be_proxy, update_version): base_file_location = os.getcwd() + os.path.sep logger.debug("working directory =" + base_file_location) - process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy) - process_type_list(get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version) - process_element_list(get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy) + model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy)) + try: + model_import_manager.create_models() + except Exception as ex: + logger.log("An error has occurred while uploading the models: ", str(ex)) + raise ex + + process_element_list(normativeElementsList.get_normative_element_candidate_list(base_file_location), sdc_be_proxy) + process_type_list(normativeTypesList.get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version) + process_element_list(normativeElementsList.get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy) logger.log("Script end ->", "All normatives imported successfully!") logger.print_and_exit(0, None) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py new file mode 100644 index 0000000000..29b01bbdee --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py @@ -0,0 +1,81 @@ +# ============LICENSE_START======================================================= +# Copyright (C) 2021 Nordix Foundation +# =============================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END======================================================== + +import json +from io import BytesIO +from pathlib import Path + +import pycurl + +from sdcBePy.common import logger + + +class ModelClient: + + def __init__(self, sdc_be_proxy): + self.__base_path = Path('/sdc2/rest/v1/catalog/model') + self.__imports_path = self.__base_path / 'imports' + self.__sdc_be_proxy = sdc_be_proxy + + def create_model(self, model_payload_dict, model_imports_zip_path): + model_name = model_payload_dict['name'] + + logger.debug("Starting to create model '{}', zip path '{}'".format(model_name, model_imports_zip_path)) + + multi_part_form_data = [] + + model_zip_param = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path))) + multi_part_form_data.append(model_zip_param) + + json_payload = self.__parse_to_json_str(model_payload_dict) + model_param = ('model', ( + pycurl.FORM_CONTENTS, json_payload, + pycurl.FORM_CONTENTTYPE, 'application/json' + )) + multi_part_form_data.append(model_param) + + response_buffer = BytesIO() + response_code = self.__sdc_be_proxy.post_file(str(self.__base_path), multi_part_form_data, response_buffer) + logger.debug("Create model response code '{}'".format(response_code)) + if response_code != 201: + error_msg = "Failed to create model '{}'".format(model_name) + logger.log(error_msg, response_buffer.getvalue()) + raise Exception(error_msg) + logger.log("Created model", model_name) + + def update_model_imports(self, model_payload_dict, model_imports_zip_path): + model_name = model_payload_dict['name'] + logger.debug("Starting to update model '{}', zip path '{}'".format(model_name, model_imports_zip_path)) + + multi_part_form_data = [] + + model_zip_post = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path))) + multi_part_form_data.append(('modelName', model_name)) + multi_part_form_data.append(model_zip_post) + + response_buffer = BytesIO() + response_code = self.__sdc_be_proxy.put_file(str(self.__imports_path), multi_part_form_data, response_buffer) + logger.debug("Update model response code '{}'".format(response_code)) + if response_code != 204: + error_msg = "Failed to update model '{}'".format(model_name) + logger.log(error_msg, response_buffer.getvalue()) + raise Exception(error_msg) + logger.log("Updated model", model_name) + + @staticmethod + def __parse_to_json_str(model_payload_dict): + return json.dumps(model_payload_dict) diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py new file mode 100644 index 0000000000..016de03b29 --- /dev/null +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py @@ -0,0 +1,91 @@ +# ============LICENSE_START======================================================= +# Copyright (C) 2021 Nordix Foundation +# =============================================================================== +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END======================================================== + +import json +import os +import zipfile +from pathlib import Path + + +class ModelImportManager: + """Manages the model imports directory""" + + INIT_FOLDER_NAME = 'init' + UPGRADE_FOLDER_NAME = 'upgrade' + IMPORTS_FOLDER_NAME = 'imports' + ACTION_UPGRADE = 'upgrade' + ACTION_INIT = 'init' + + def __init__(self, model_imports_path, model_client): + self.__model_base_path = model_imports_path + self.__model_init_path = self.__model_base_path / self.INIT_FOLDER_NAME + self.__model_upgrade_path = self.__model_base_path / self.UPGRADE_FOLDER_NAME + self.__model_client = model_client + + def create_models(self): + for model_folder_name in self.__get_model_init_list(): + model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT) + model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT) + self.__model_client.create_model(model_payload_dict, model_imports_zip_path) + + def update_models(self): + for model_folder_name in self.__get_model_upgrade_list(): + model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE) + model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE) + self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path) + + def __get_model_init_list(self): + return self.__get_model_list(self.__model_init_path) + + def __get_model_upgrade_list(self): + return self.__get_model_list(self.__model_upgrade_path) + + @staticmethod + def __get_model_list(path): + model_list = [] + for (dirpath, dirnames, filenames) in os.walk(path): + model_list.extend(dirnames) + break + return model_list + + def __zip_model_imports(self, model, action_type) -> Path: + base_path = self.__get_base_action_path(action_type) + model_path = base_path / model + model_imports_path = base_path / model / self.IMPORTS_FOLDER_NAME + zip_file_path = model_path / "{}.zip".format(model) + zip_file = zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) + for root, dirs, files in os.walk(model_imports_path): + for file in files: + zip_file.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), model_imports_path)) + zip_file.close() + return zip_file_path + + def __read_model_payload_as_string(self, model, action_type) -> str: + base_path = self.__get_base_action_path(action_type) + model_payload_path = base_path / model / "payload.json" + json_file = open(model_payload_path) + json_data = json.load(json_file, strict=False) + return json.dumps(json_data) + + def __read_model_payload(self, model, action_type) -> dict: + base_path = self.__get_base_action_path(action_type) + model_payload_path = base_path / model / "payload.json" + json_file = open(model_payload_path) + return json.load(json_file, strict=False) + + def __get_base_action_path(self, action_type) -> Path: + return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py index cef7dd74db..44a25a0136 100644 --- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py +++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 import os +from pathlib import Path from sdcBePy.common import logger from sdcBePy.common.logger import print_and_exit from sdcBePy.common.normative.main import process_element_list, process_type_list from sdcBePy.tosca.main import parse_and_create_proxy +from sdcBePy.tosca.models.model_client import ModelClient +from sdcBePy.tosca.models.model_import_manager import ModelImportManager from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \ get_normative_element_with_metadata_list from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \ @@ -21,6 +24,14 @@ def main(sdc_be_proxy): # base_file_location = os.getcwd() + "/../../../../import/tosca/" base_file_location = os.getcwd() + "/" logger.debug("working directory =" + base_file_location) + + model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy)) + try: + model_import_manager.update_models() + except Exception as ex: + logger.log("An error has occurred while uploading the models: ", str(ex)) + raise ex + process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy) all_types = get_all_types() -- cgit 1.2.3-korg