summaryrefslogtreecommitdiffstats
path: root/catalog-be/src/main/resources/scripts
diff options
context:
space:
mode:
authorandre.schmid <andre.schmid@est.tech>2021-05-28 19:10:30 +0100
committerChristophe Closset <christophe.closset@intl.att.com>2021-06-14 08:19:21 +0000
commitddf9aaefc753b492fb72144d597a27df8080a4ab (patch)
tree8d998a1287b903f6669df8568bdb6d2edda60eac /catalog-be/src/main/resources/scripts
parentc82aebcde26e34c4151531b4d7a8f6e7689734ba (diff)
Init ONAP model imports using the model API
Creates a client for the model endpoint in the catalog init scripts. Introduces the directory structure to provide the models along its imports, separated by init/upgrade phase. Each model structure will be zipped and uploaded to the endpoint, based on the model directory name. Change-Id: I0392c1e6d3a29b30567b11016041a8e9cccbc745 Issue-ID: SDC-3615 Signed-off-by: André Schmid <andre.schmid@est.tech>
Diffstat (limited to 'catalog-be/src/main/resources/scripts')
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py1
-rwxr-xr-xcatalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py74
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py21
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py81
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py91
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py11
6 files changed, 245 insertions, 34 deletions
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
index 5ef3173907..654a755863 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
@@ -47,6 +47,7 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name,
multi_part_form_data = _create_multipart_form_data(element_form_name, type_file_name, with_metadata,
element_name)
+ debug("http request url =", url_suffix)
http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data)
if http_res is not None:
debug("http response =", http_res)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
index 169979ae73..f2bef53564 100755
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
@@ -21,7 +21,7 @@ class SdcBeProxy:
raise AttributeError("The be_host, be_port, scheme or admin_user are missing")
url = get_url(be_ip, be_port, scheme)
self.con = connector if connector \
- else CurlConnector(url, user_id, header, scheme=scheme, debug=debug)
+ else CurlConnector(url, user_id, header, protocol=scheme, debug=debug)
def check_backend(self):
return self.con.get('/sdc2/rest/v1/user/jh0003')
@@ -52,8 +52,11 @@ class SdcBeProxy:
def get_normatives(self):
return self.con.get("/sdc2/rest/v1/screen", with_buffer=True)
- def post_file(self, path, multi_part_form_data):
- return self.con.post_file(path, multi_part_form_data)
+ def post_file(self, path, multi_part_form_data, buffer=None):
+ return self.con.post_file(path, multi_part_form_data, buffer)
+
+ def put_file(self, path, multi_part_form_data, buffer=None):
+ return self.con.put_file(path, multi_part_form_data, buffer)
def get_response_from_buffer(self):
value = self.con.buffer.getvalue()
@@ -68,17 +71,13 @@ class CurlConnector:
CONTENT_TYPE_HEADER = "Content-Type: application/json"
ACCEPT_HEADER = "Accept: application/json; charset=UTF-8"
- def __init__(self, url, user_id_header, header, buffer=None, scheme="http", debug=False):
- self.c = pycurl.Curl()
- self.c.setopt(pycurl.HEADER, True)
+ def __init__(self, url, user_id_header, header, buffer=None, protocol="http", debug=False):
+ self.__debug = debug
+ self.__protocol = protocol
+ self.c = self.__build_default_curl()
self.user_header = "USER_ID: " + user_id_header
-
- if not debug:
- # disable printing not necessary logs in the terminal
- self.c.setopt(pycurl.WRITEFUNCTION, lambda x: None)
- else:
- self.c.setopt(pycurl.VERBOSE, 1)
+ self.url = url
if not buffer:
self.buffer = BytesIO()
@@ -88,9 +87,6 @@ class CurlConnector:
else:
self.basicauth_header = "Authorization: Basic " + header
- self.url = url
- self._check_schema(scheme)
-
def get(self, path, buffer=None, with_buffer=False):
try:
self.c.setopt(pycurl.URL, self.url + path)
@@ -99,7 +95,6 @@ class CurlConnector:
CurlConnector.ACCEPT_HEADER,
self.basicauth_header])
-
if with_buffer:
write = self.buffer.write if not buffer else buffer.write
self.c.setopt(pycurl.WRITEFUNCTION, write)
@@ -115,9 +110,9 @@ class CurlConnector:
self.c.setopt(pycurl.POST, 1)
self.c.setopt(pycurl.HTTPHEADER, [self.user_header,
- CurlConnector.CONTENT_TYPE_HEADER,
- CurlConnector.ACCEPT_HEADER,
- self.basicauth_header])
+ CurlConnector.CONTENT_TYPE_HEADER,
+ CurlConnector.ACCEPT_HEADER,
+ self.basicauth_header])
self.c.setopt(pycurl.POSTFIELDS, data)
@@ -132,9 +127,7 @@ class CurlConnector:
try:
self.c.setopt(pycurl.URL, self.url + path)
self.c.setopt(pycurl.POST, 1)
- self.c.setopt(pycurl.HTTPHEADER, [self.user_header,
- self.basicauth_header])
-
+ self.c.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header])
self.c.setopt(pycurl.HTTPPOST, post_body)
@@ -143,15 +136,40 @@ class CurlConnector:
self.c.perform()
self.c.setopt(pycurl.POST, 0)
-
return self.c.getinfo(pycurl.RESPONSE_CODE)
- except pycurl.error:
+ except pycurl.error as ex:
+ print(ex)
return 111
- def _check_schema(self, scheme):
- if scheme == 'https':
- self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
- self.c.setopt(pycurl.SSL_VERIFYHOST, 0)
+ def put_file(self, path, post_body, response_write_buffer=None):
+ curl = self.__build_default_curl()
+ curl.setopt(pycurl.URL, self.url + path)
+ curl.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header])
+ curl.setopt(pycurl.CUSTOMREQUEST, "PUT")
+
+ curl.setopt(pycurl.HTTPPOST, post_body)
+
+ write = self.buffer.write if not response_write_buffer else response_write_buffer.write
+ curl.setopt(pycurl.WRITEFUNCTION, write)
+
+ curl.perform()
+ response_code = curl.getinfo(pycurl.RESPONSE_CODE)
+ curl.close()
+ return response_code
+
+ def __build_default_curl(self):
+ curl = pycurl.Curl()
+ if not self.__debug:
+ # disable printing not necessary logs in the terminal
+ curl.setopt(pycurl.WRITEFUNCTION, lambda x: None)
+ else:
+ curl.setopt(pycurl.VERBOSE, 1)
+
+ if self.__protocol == 'https':
+ curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+ curl.setopt(pycurl.SSL_VERIFYHOST, 0)
+ curl.setopt(pycurl.HEADER, True)
+ return curl
def __del__(self):
self.c.close()
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
index 9ac820071a..067f110efc 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
@@ -1,13 +1,15 @@
#!/usr/bin/env python3
import os
+from pathlib import Path
import sdcBePy.common.logger as logger
from sdcBePy.common.normative.main import process_element_list, process_type_list
from sdcBePy.tosca.main import parse_and_create_proxy
-from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
- get_normative_element_with_metadata_list
-from sdcBePy.tosca.models.normativeTypesList import get_normative_type_candidate_list
+from sdcBePy.tosca.models import normativeElementsList
+from sdcBePy.tosca.models import normativeTypesList
+from sdcBePy.tosca.models.model_client import ModelClient
+from sdcBePy.tosca.models.model_import_manager import ModelImportManager
def main(sdc_be_proxy, update_version):
@@ -16,9 +18,16 @@ def main(sdc_be_proxy, update_version):
base_file_location = os.getcwd() + os.path.sep
logger.debug("working directory =" + base_file_location)
- process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
- process_type_list(get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
- process_element_list(get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
+ model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+ try:
+ model_import_manager.create_models()
+ except Exception as ex:
+ logger.log("An error has occurred while uploading the models: ", str(ex))
+ raise ex
+
+ process_element_list(normativeElementsList.get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
+ process_type_list(normativeTypesList.get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
+ process_element_list(normativeElementsList.get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
logger.log("Script end ->", "All normatives imported successfully!")
logger.print_and_exit(0, None)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
new file mode 100644
index 0000000000..29b01bbdee
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
@@ -0,0 +1,81 @@
+# ============LICENSE_START=======================================================
+# Copyright (C) 2021 Nordix Foundation
+# ===============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END========================================================
+
+import json
+from io import BytesIO
+from pathlib import Path
+
+import pycurl
+
+from sdcBePy.common import logger
+
+
+class ModelClient:
+
+ def __init__(self, sdc_be_proxy):
+ self.__base_path = Path('/sdc2/rest/v1/catalog/model')
+ self.__imports_path = self.__base_path / 'imports'
+ self.__sdc_be_proxy = sdc_be_proxy
+
+ def create_model(self, model_payload_dict, model_imports_zip_path):
+ model_name = model_payload_dict['name']
+
+ logger.debug("Starting to create model '{}', zip path '{}'".format(model_name, model_imports_zip_path))
+
+ multi_part_form_data = []
+
+ model_zip_param = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path)))
+ multi_part_form_data.append(model_zip_param)
+
+ json_payload = self.__parse_to_json_str(model_payload_dict)
+ model_param = ('model', (
+ pycurl.FORM_CONTENTS, json_payload,
+ pycurl.FORM_CONTENTTYPE, 'application/json'
+ ))
+ multi_part_form_data.append(model_param)
+
+ response_buffer = BytesIO()
+ response_code = self.__sdc_be_proxy.post_file(str(self.__base_path), multi_part_form_data, response_buffer)
+ logger.debug("Create model response code '{}'".format(response_code))
+ if response_code != 201:
+ error_msg = "Failed to create model '{}'".format(model_name)
+ logger.log(error_msg, response_buffer.getvalue())
+ raise Exception(error_msg)
+ logger.log("Created model", model_name)
+
+ def update_model_imports(self, model_payload_dict, model_imports_zip_path):
+ model_name = model_payload_dict['name']
+ logger.debug("Starting to update model '{}', zip path '{}'".format(model_name, model_imports_zip_path))
+
+ multi_part_form_data = []
+
+ model_zip_post = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path)))
+ multi_part_form_data.append(('modelName', model_name))
+ multi_part_form_data.append(model_zip_post)
+
+ response_buffer = BytesIO()
+ response_code = self.__sdc_be_proxy.put_file(str(self.__imports_path), multi_part_form_data, response_buffer)
+ logger.debug("Update model response code '{}'".format(response_code))
+ if response_code != 204:
+ error_msg = "Failed to update model '{}'".format(model_name)
+ logger.log(error_msg, response_buffer.getvalue())
+ raise Exception(error_msg)
+ logger.log("Updated model", model_name)
+
+ @staticmethod
+ def __parse_to_json_str(model_payload_dict):
+ return json.dumps(model_payload_dict)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
new file mode 100644
index 0000000000..016de03b29
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
@@ -0,0 +1,91 @@
+# ============LICENSE_START=======================================================
+# Copyright (C) 2021 Nordix Foundation
+# ===============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END========================================================
+
+import json
+import os
+import zipfile
+from pathlib import Path
+
+
+class ModelImportManager:
+ """Manages the model imports directory"""
+
+ INIT_FOLDER_NAME = 'init'
+ UPGRADE_FOLDER_NAME = 'upgrade'
+ IMPORTS_FOLDER_NAME = 'imports'
+ ACTION_UPGRADE = 'upgrade'
+ ACTION_INIT = 'init'
+
+ def __init__(self, model_imports_path, model_client):
+ self.__model_base_path = model_imports_path
+ self.__model_init_path = self.__model_base_path / self.INIT_FOLDER_NAME
+ self.__model_upgrade_path = self.__model_base_path / self.UPGRADE_FOLDER_NAME
+ self.__model_client = model_client
+
+ def create_models(self):
+ for model_folder_name in self.__get_model_init_list():
+ model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT)
+ model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT)
+ self.__model_client.create_model(model_payload_dict, model_imports_zip_path)
+
+ def update_models(self):
+ for model_folder_name in self.__get_model_upgrade_list():
+ model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE)
+ model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE)
+ self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path)
+
+ def __get_model_init_list(self):
+ return self.__get_model_list(self.__model_init_path)
+
+ def __get_model_upgrade_list(self):
+ return self.__get_model_list(self.__model_upgrade_path)
+
+ @staticmethod
+ def __get_model_list(path):
+ model_list = []
+ for (dirpath, dirnames, filenames) in os.walk(path):
+ model_list.extend(dirnames)
+ break
+ return model_list
+
+ def __zip_model_imports(self, model, action_type) -> Path:
+ base_path = self.__get_base_action_path(action_type)
+ model_path = base_path / model
+ model_imports_path = base_path / model / self.IMPORTS_FOLDER_NAME
+ zip_file_path = model_path / "{}.zip".format(model)
+ zip_file = zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED)
+ for root, dirs, files in os.walk(model_imports_path):
+ for file in files:
+ zip_file.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), model_imports_path))
+ zip_file.close()
+ return zip_file_path
+
+ def __read_model_payload_as_string(self, model, action_type) -> str:
+ base_path = self.__get_base_action_path(action_type)
+ model_payload_path = base_path / model / "payload.json"
+ json_file = open(model_payload_path)
+ json_data = json.load(json_file, strict=False)
+ return json.dumps(json_data)
+
+ def __read_model_payload(self, model, action_type) -> dict:
+ base_path = self.__get_base_action_path(action_type)
+ model_payload_path = base_path / model / "payload.json"
+ json_file = open(model_payload_path)
+ return json.load(json_file, strict=False)
+
+ def __get_base_action_path(self, action_type) -> Path:
+ return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
index cef7dd74db..44a25a0136 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
@@ -1,11 +1,14 @@
#!/usr/bin/env python3
import os
+from pathlib import Path
from sdcBePy.common import logger
from sdcBePy.common.logger import print_and_exit
from sdcBePy.common.normative.main import process_element_list, process_type_list
from sdcBePy.tosca.main import parse_and_create_proxy
+from sdcBePy.tosca.models.model_client import ModelClient
+from sdcBePy.tosca.models.model_import_manager import ModelImportManager
from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
get_normative_element_with_metadata_list
from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \
@@ -21,6 +24,14 @@ def main(sdc_be_proxy):
# base_file_location = os.getcwd() + "/../../../../import/tosca/"
base_file_location = os.getcwd() + "/"
logger.debug("working directory =" + base_file_location)
+
+ model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+ try:
+ model_import_manager.update_models()
+ except Exception as ex:
+ logger.log("An error has occurred while uploading the models: ", str(ex))
+ raise ex
+
process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
all_types = get_all_types()