summaryrefslogtreecommitdiffstats
path: root/catalog-be/src/main/resources/scripts
diff options
context:
space:
mode:
authorandre.schmid <andre.schmid@est.tech>2021-10-01 18:45:17 +0100
committerAndré Schmid <andre.schmid@est.tech>2021-10-15 14:26:03 +0000
commit1b61e72f6506de7c1f64098dd25adf7af7b31dc1 (patch)
tree53036d7bb556547059f1207ec7c8a42959cd05a7 /catalog-be/src/main/resources/scripts
parent08ba15073ff236c67303d084d56a026975ba1a25 (diff)
Import multiple node types in a single endpoint
Change-Id: I7aa914d459a083146b92ef82197c8df265b8c5b6 Issue-ID: SDC-3752 Signed-off-by: andre.schmid <andre.schmid@est.tech>
Diffstat (limited to 'catalog-be/src/main/resources/scripts')
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py7
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py10
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py4
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py7
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py61
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py75
-rw-r--r--catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py4
7 files changed, 142 insertions, 26 deletions
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
index 17ef9afafb..2508d9d9c4 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
@@ -12,8 +12,11 @@ def debug(desc, *args):
print(desc, join_strings(args))
-def log(desc, arg):
- print(desc, arg)
+def log(desc, arg=None):
+ if arg:
+ print(desc, arg)
+ else:
+ print(desc)
def print_and_exit(error_code, error_desc):
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
index 56fe2b4b39..9304c88b44 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
@@ -31,13 +31,13 @@ def print_and_check_results(results, update_version, exit_on_success=False):
for result in results:
print_name_and_return_code(result[0], result[1], with_line=False)
print("----------------------------------------")
- check_results_and_exit(results, update_version, exit_on_success)
+ check_results_and_exit(results, exit_on_success)
else:
raise ResourceCreationError("Results is none -> error occurred!!", 1)
-def check_results_and_exit(results, update_version, exit_on_success):
- if not _results_ok(results, _get_response_code(update_version)):
+def check_results_and_exit(results, exit_on_success):
+ if not _results_ok(results, _get_response_code()):
raise ResourceCreationError("Failed to create the normatives types !!", 1)
else:
if exit_on_success:
@@ -46,7 +46,7 @@ def check_results_and_exit(results, update_version, exit_on_success):
def _create_normatives_type(file_dir, sdc_be_proxy, types, update_version):
results = []
- response_codes = _get_response_code(update_version)
+ response_codes = _get_response_code()
for normative_type in types:
result = _send_request(sdc_be_proxy, file_dir, normative_type, update_version)
results.append(result)
@@ -113,7 +113,7 @@ def _results_ok(results, response_codes):
return True
-def _get_response_code(update_version):
+def _get_response_code():
response_codes = [200, 201, 409]
return response_codes
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
index 39e802bf87..db6e91481c 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
@@ -10,6 +10,7 @@ from sdcBePy.tosca.models import normativeElementsList
from sdcBePy.tosca.models import normativeTypesList
from sdcBePy.tosca.models.model_client import ModelClient
from sdcBePy.tosca.models.model_import_manager import ModelImportManager
+from sdcBePy.tosca.models.node_type_client import NodeTypeClient
def main(sdc_be_proxy, update_version):
@@ -24,7 +25,8 @@ def main(sdc_be_proxy, update_version):
process_type_list(normativeTypesList.get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
process_element_list(normativeElementsList.get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
#Add model based normatives
- model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+ model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy),
+ NodeTypeClient(sdc_be_proxy))
model_import_manager.deploy_models()
except Exception as ex:
logger.log("An error has occurred while uploading elements and types: ", str(ex))
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
index 88128ea71d..8ba05f6c31 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
@@ -23,7 +23,9 @@ import pycurl
from sdcBePy.common import logger
from sdcBePy.common.normative.main import process_element_list, process_type_list
-from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, get_normative_element_with_metadata_list
+from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
+ get_normative_element_with_metadata_list
+
class ModelClient:
@@ -86,8 +88,7 @@ class ModelClient:
process_element_list(get_normative_element_candidate_list(tosca_elements_import_path), self.__sdc_be_proxy, model=model_name)
logger.log("Finished importing normative elements for model", model_name)
- def import_model_types(self, model_payload_dict, types_list, upgrade):
- model_name = model_payload_dict['name']
+ def import_model_types(self, model_name, types_list, upgrade):
logger.debug("Starting import of normative types for model '{}'".format(model_name))
process_type_list(types_list, self.__sdc_be_proxy, upgrade)
logger.log("Finished importing normative types for model", model_name)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
index 57b0f1399f..57ac06d0b2 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
@@ -19,6 +19,7 @@ import json
import os
import zipfile
from sdcBePy.tosca.models.normativeTypeCandidate import NormativeTypeCandidate
+from sdcBePy.common import logger
from pathlib import Path
@@ -32,37 +33,41 @@ class ModelImportManager:
ACTION_INIT = 'init'
TYPES_FOLDER = 'tosca'
NODE_FOLDER = 'node-types'
+ NODE_TYPE_FILE = 'nodeTypes.yaml'
- def __init__(self, model_imports_path, model_client):
+ def __init__(self, model_imports_path, model_client, node_type_client):
self.__model_base_path = model_imports_path
self.__model_init_path = self.__model_base_path / self.INIT_FOLDER_NAME
self.__model_upgrade_path = self.__model_base_path / self.UPGRADE_FOLDER_NAME
self.__model_client = model_client
+ self.__node_type_client = node_type_client
def deploy_models(self):
existing_models = self.__model_client.get_model_list()
for model_folder_name in self.__get_model_init_list():
model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT)
- if (not existing_models or not any(m for m in existing_models if model_payload_dict['name'] == m['name'])):
+ if not existing_models or not any(m for m in existing_models if model_payload_dict['name'] == m['name']):
self.__create_models(model_folder_name, model_payload_dict)
for model_folder_name in self.__get_model_upgrade_list():
model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE)
- if (existing_models and any(m for m in existing_models if model_payload_dict['name'] == m['name'])):
+ if existing_models and any(m for m in existing_models if model_payload_dict['name'] == m['name']):
self.__update_models(model_folder_name, model_payload_dict)
def __create_models(self, model_folder_name, model_payload_dict):
+ logger.log('Creating model {}, based on folder {}'.format(model_payload_dict['name'], model_folder_name))
model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT)
self.__model_client.create_model(model_payload_dict, model_imports_zip_path)
self.__init_model_non_node_types(model_folder_name, model_payload_dict)
- self.__init_model_node_types(model_folder_name, model_payload_dict)
- self.__init_model_non_node_types(model_folder_name, model_payload_dict, True);
+ self.__init_model_node_types(model_folder_name, model_payload_dict['name'])
+ self.__init_model_non_node_types(model_folder_name, model_payload_dict, True)
def __update_models(self, model_folder_name, model_payload_dict):
+ logger.log('Updating model {}, based on folder {}'.format(model_payload_dict['name'], model_folder_name))
model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE)
self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path)
self.__upgrade_model_non_node_types(model_folder_name, model_payload_dict)
- self.__upgrade_model_node_types(model_folder_name, model_payload_dict)
+ self.__upgrade_model_node_types(model_folder_name, model_payload_dict['name'])
self.__upgrade_model_non_node_types(model_folder_name, model_payload_dict, True)
def __get_model_init_list(self):
@@ -118,6 +123,8 @@ class ModelImportManager:
return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path
def __get_tosca_path(self, action, model):
+ if action not in [self.ACTION_INIT, self.ACTION_UPGRADE]:
+ raise Exception("Invalid action {}. Expected {}".format(action, [self.ACTION_INIT, self.ACTION_UPGRADE]))
return self.__get_base_action_path(action) / model / self.TYPES_FOLDER
def __init_model_non_node_types(self, model, model_payload_dict, with_metadata=False):
@@ -130,12 +137,38 @@ class ModelImportManager:
if os.path.isdir(path):
self.__model_client.import_model_elements(model_payload_dict, str(os.path.join(path, '')), with_metadata)
- def __init_model_node_types(self, model, model_payload_dict, upgrade=False):
- path = self.__get_tosca_path(self.ACTION_INIT, model) / self.NODE_FOLDER
- if os.path.isdir(path):
- self.__model_client.import_model_types(model_payload_dict, self.__get_node_type_list(path), upgrade)
+ def __init_model_node_types(self, model_folder_name, model_name):
+ self.__import_model_node_types(model_folder_name, model_name, self.ACTION_INIT)
- def __upgrade_model_node_types(self, model, model_payload_dict, upgrade=True):
- path = self.__get_tosca_path(self.ACTION_UPGRADE, model) / self.NODE_FOLDER
- if os.path.isdir(path):
- self.__model_client.import_model_types(model_payload_dict, self.__get_node_type_list(path), upgrade)
+ def __upgrade_model_node_types(self, model_folder_name, model_name):
+ self.__import_model_node_types(model_folder_name, model_name, self.ACTION_UPGRADE)
+
+ def __import_model_node_types(self, model_folder_name, model_name, action):
+ path = self.__get_tosca_path(action, model_folder_name) / self.NODE_FOLDER
+ if not os.path.isdir(path):
+ return
+
+ payload_json_path = self.__get_node_types_metadata_file_path(model_name, path)
+ payload_json_str = json.dumps(json.load(open(payload_json_path)))
+
+ node_types_yaml_path = self.__get_node_types_yaml_file_path(model_name, path)
+
+ is_update = True if action == self.ACTION_UPGRADE else False
+
+ self.__node_type_client.import_all(node_types_yaml_path, payload_json_str, is_update)
+
+ def __get_node_types_metadata_file_path(self, model_name, node_types_folder):
+ metadata_json_path = node_types_folder / 'metadata.json'
+ if not os.path.isfile(metadata_json_path):
+ error_msg = "Missing metadata.json file for model '{}'. Expected path '{}'".format(model_name,
+ metadata_json_path)
+ raise Exception(error_msg)
+ return metadata_json_path
+
+ def __get_node_types_yaml_file_path(self, model_name, node_types_folder):
+ node_types_yaml_path = node_types_folder / self.NODE_TYPE_FILE
+ if not os.path.isfile(node_types_yaml_path):
+ error_msg = "Missing {} file for model '{}'. Expected path '{}'".format(self.NODE_TYPE_FILE, model_name,
+ node_types_yaml_path)
+ raise Exception(error_msg)
+ return node_types_yaml_path
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py
new file mode 100644
index 0000000000..7937002567
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py
@@ -0,0 +1,75 @@
+# -
+# ============LICENSE_START=======================================================
+# Copyright (C) 2021 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END========================================================
+
+from io import BytesIO
+from pathlib import Path
+
+import pycurl
+
+from sdcBePy.common import logger
+
+
+class NodeTypeClient:
+
+ def __init__(self, sdc_be_proxy):
+ self.__base_path = Path('/sdc2/rest/v1/catalog/upload')
+ self.__import_all_path = self.__base_path / 'resource/import'
+ self.__sdc_be_proxy = sdc_be_proxy
+
+ def import_all(self, node_type_yaml_path, node_type_metadata_json_str, is_update=False):
+ logger.debug("Starting to import node types '{}'".format(node_type_yaml_path))
+
+ multi_part_form_data = []
+
+ node_type_yaml_param = ('nodeTypesYaml', (pycurl.FORM_FILE, str(node_type_yaml_path)))
+ multi_part_form_data.append(node_type_yaml_param)
+
+ node_type_metadata_json_param = ('nodeTypeMetadataJson', (
+ pycurl.FORM_CONTENTS, node_type_metadata_json_str,
+ pycurl.FORM_CONTENTTYPE, 'application/json'
+ ))
+ multi_part_form_data.append(node_type_metadata_json_param)
+
+ if is_update is not None:
+ create_new_version_param = ('createNewVersion', (
+ pycurl.FORM_CONTENTS, str(is_update).lower(),
+ pycurl.FORM_CONTENTTYPE, 'text/plain'
+ ))
+ multi_part_form_data.append(create_new_version_param)
+
+ response_buffer = BytesIO()
+ response_code = self.__sdc_be_proxy.post_file(str(self.__import_all_path), multi_part_form_data,
+ response_buffer)
+ logger.debug("Import all node types response code '{}'".format(response_code))
+ if response_code != 201:
+ error_msg = "Failed to import node types '{}'".format(node_type_yaml_path)
+ logger.log(error_msg, response_buffer.getvalue())
+ raise Exception(error_msg)
+ logger.log("Failed to import node types '{}'".format(node_type_yaml_path))
+
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
index 663d27b435..6af44920ca 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
@@ -9,6 +9,7 @@ from sdcBePy.common.normative.main import process_element_list, process_type_lis
from sdcBePy.tosca.main import parse_and_create_proxy
from sdcBePy.tosca.models.model_client import ModelClient
from sdcBePy.tosca.models.model_import_manager import ModelImportManager
+from sdcBePy.tosca.models.node_type_client import NodeTypeClient
from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
get_normative_element_with_metadata_list
from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \
@@ -25,7 +26,8 @@ def main(sdc_be_proxy):
base_file_location = os.getcwd() + "/"
logger.debug("working directory =" + base_file_location)
- model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+ model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy),
+ NodeTypeClient(sdc_be_proxy))
try:
model_import_manager.deploy_models()
except Exception as ex: