Import multiple node types in a single endpoint

Change-Id: I7aa914d459a083146b92ef82197c8df265b8c5b6
Issue-ID: SDC-3752
Signed-off-by: andre.schmid <andre.schmid@est.tech>

7 files changed, 142 insertions, 26 deletions

diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
index 17ef9afafb..2508d9d9c4 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/logger.py
@@ -12,8 +12,11 @@ def debug(desc, *args):
         print(desc, join_strings(args))
 
 
-def log(desc, arg):
-    print(desc, arg)
+def log(desc, arg=None):
+    if arg:
+        print(desc, arg)
+    else:
+        print(desc)
 
 
 def print_and_exit(error_code, error_desc):
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
index 56fe2b4b39..9304c88b44 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaTypes.py
@@ -31,13 +31,13 @@ def print_and_check_results(results, update_version, exit_on_success=False):
         for result in results:
             print_name_and_return_code(result[0], result[1], with_line=False)
         print("----------------------------------------")
-        check_results_and_exit(results, update_version, exit_on_success)
+        check_results_and_exit(results, exit_on_success)
     else:
         raise ResourceCreationError("Results is none -> error occurred!!", 1)
 
 
-def check_results_and_exit(results, update_version, exit_on_success):
-    if not _results_ok(results, _get_response_code(update_version)):
+def check_results_and_exit(results, exit_on_success):
+    if not _results_ok(results, _get_response_code()):
         raise ResourceCreationError("Failed to create the normatives types !!", 1)
     else:
         if exit_on_success:
@@ -46,7 +46,7 @@ def check_results_and_exit(results, update_version, exit_on_success):
 
 def _create_normatives_type(file_dir, sdc_be_proxy, types, update_version):
     results = []
-    response_codes = _get_response_code(update_version)
+    response_codes = _get_response_code()
     for normative_type in types:
         result = _send_request(sdc_be_proxy, file_dir, normative_type, update_version)
         results.append(result)
@@ -113,7 +113,7 @@ def _results_ok(results, response_codes):
     return True
 
 
-def _get_response_code(update_version):
+def _get_response_code():
     response_codes = [200, 201, 409]
     return response_codes
 
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
index 39e802bf87..db6e91481c 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
@@ -10,6 +10,7 @@ from sdcBePy.tosca.models import normativeElementsList
 from sdcBePy.tosca.models import normativeTypesList
 from sdcBePy.tosca.models.model_client import ModelClient
 from sdcBePy.tosca.models.model_import_manager import ModelImportManager
+from sdcBePy.tosca.models.node_type_client import NodeTypeClient
 
 
 def main(sdc_be_proxy, update_version):
@@ -24,7 +25,8 @@ def main(sdc_be_proxy, update_version):
         process_type_list(normativeTypesList.get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
         process_element_list(normativeElementsList.get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
         #Add model based normatives
-        model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+        model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy),
+                                                  NodeTypeClient(sdc_be_proxy))
         model_import_manager.deploy_models()
     except Exception as ex:
         logger.log("An error has occurred while uploading elements and types: ", str(ex))
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
index 88128ea71d..8ba05f6c31 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
@@ -23,7 +23,9 @@ import pycurl
 
 from sdcBePy.common import logger
 from sdcBePy.common.normative.main import process_element_list, process_type_list
-from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, get_normative_element_with_metadata_list
+from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
+    get_normative_element_with_metadata_list
+
 
 class ModelClient:
 
@@ -86,8 +88,7 @@ class ModelClient:
             process_element_list(get_normative_element_candidate_list(tosca_elements_import_path), self.__sdc_be_proxy, model=model_name)
         logger.log("Finished importing normative elements for model", model_name)
 
-    def import_model_types(self, model_payload_dict, types_list, upgrade):
-        model_name = model_payload_dict['name']
+    def import_model_types(self, model_name, types_list, upgrade):
         logger.debug("Starting import of normative types for model '{}'".format(model_name))
         process_type_list(types_list, self.__sdc_be_proxy, upgrade)
         logger.log("Finished importing normative types for model", model_name)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
index 57b0f1399f..57ac06d0b2 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
@@ -19,6 +19,7 @@ import json
 import os
 import zipfile
 from sdcBePy.tosca.models.normativeTypeCandidate import NormativeTypeCandidate
+from sdcBePy.common import logger
 from pathlib import Path
 
 
@@ -32,37 +33,41 @@ class ModelImportManager:
     ACTION_INIT = 'init'
     TYPES_FOLDER = 'tosca'
     NODE_FOLDER = 'node-types'
+    NODE_TYPE_FILE = 'nodeTypes.yaml'
 
-    def __init__(self, model_imports_path, model_client):
+    def __init__(self, model_imports_path, model_client, node_type_client):
         self.__model_base_path = model_imports_path
         self.__model_init_path = self.__model_base_path / self.INIT_FOLDER_NAME
         self.__model_upgrade_path = self.__model_base_path / self.UPGRADE_FOLDER_NAME
         self.__model_client = model_client
+        self.__node_type_client = node_type_client
 
     def deploy_models(self):
         existing_models = self.__model_client.get_model_list()
         for model_folder_name in self.__get_model_init_list():
             model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT)
-            if (not existing_models or not any(m for m in existing_models if model_payload_dict['name'] == m['name'])):
+            if not existing_models or not any(m for m in existing_models if model_payload_dict['name'] == m['name']):
                 self.__create_models(model_folder_name, model_payload_dict)
 
         for model_folder_name in self.__get_model_upgrade_list():
             model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE)
-            if (existing_models and any(m for m in existing_models if model_payload_dict['name'] == m['name'])):
+            if existing_models and any(m for m in existing_models if model_payload_dict['name'] == m['name']):
                 self.__update_models(model_folder_name, model_payload_dict)
 
     def __create_models(self, model_folder_name, model_payload_dict):
+        logger.log('Creating model {}, based on folder {}'.format(model_payload_dict['name'], model_folder_name))
         model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT)
         self.__model_client.create_model(model_payload_dict, model_imports_zip_path)
         self.__init_model_non_node_types(model_folder_name, model_payload_dict)
-        self.__init_model_node_types(model_folder_name, model_payload_dict)
-        self.__init_model_non_node_types(model_folder_name, model_payload_dict, True);
+        self.__init_model_node_types(model_folder_name, model_payload_dict['name'])
+        self.__init_model_non_node_types(model_folder_name, model_payload_dict, True)
 
     def __update_models(self, model_folder_name, model_payload_dict):
+        logger.log('Updating model {}, based on folder {}'.format(model_payload_dict['name'], model_folder_name))
         model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE)
         self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path)
         self.__upgrade_model_non_node_types(model_folder_name, model_payload_dict)
-        self.__upgrade_model_node_types(model_folder_name, model_payload_dict)
+        self.__upgrade_model_node_types(model_folder_name, model_payload_dict['name'])
         self.__upgrade_model_non_node_types(model_folder_name, model_payload_dict, True)
 
     def __get_model_init_list(self):
@@ -118,6 +123,8 @@ class ModelImportManager:
         return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path
 
     def __get_tosca_path(self, action, model):
+        if action not in [self.ACTION_INIT, self.ACTION_UPGRADE]:
+            raise Exception("Invalid action {}. Expected {}".format(action, [self.ACTION_INIT, self.ACTION_UPGRADE]))
         return self.__get_base_action_path(action) / model / self.TYPES_FOLDER
 
     def __init_model_non_node_types(self, model, model_payload_dict, with_metadata=False):
@@ -130,12 +137,38 @@ class ModelImportManager:
         if os.path.isdir(path):
             self.__model_client.import_model_elements(model_payload_dict, str(os.path.join(path, '')), with_metadata)
 
-    def __init_model_node_types(self, model, model_payload_dict, upgrade=False):
-        path = self.__get_tosca_path(self.ACTION_INIT, model) / self.NODE_FOLDER
-        if os.path.isdir(path):
-            self.__model_client.import_model_types(model_payload_dict, self.__get_node_type_list(path), upgrade)
+    def __init_model_node_types(self, model_folder_name, model_name):
+        self.__import_model_node_types(model_folder_name, model_name, self.ACTION_INIT)
 
-    def __upgrade_model_node_types(self, model, model_payload_dict, upgrade=True):
-        path = self.__get_tosca_path(self.ACTION_UPGRADE, model) / self.NODE_FOLDER
-        if os.path.isdir(path):
-            self.__model_client.import_model_types(model_payload_dict, self.__get_node_type_list(path), upgrade)
+    def __upgrade_model_node_types(self, model_folder_name, model_name):
+        self.__import_model_node_types(model_folder_name, model_name, self.ACTION_UPGRADE)
+
+    def __import_model_node_types(self, model_folder_name, model_name, action):
+        path = self.__get_tosca_path(action, model_folder_name) / self.NODE_FOLDER
+        if not os.path.isdir(path):
+            return
+
+        payload_json_path = self.__get_node_types_metadata_file_path(model_name, path)
+        payload_json_str = json.dumps(json.load(open(payload_json_path)))
+
+        node_types_yaml_path = self.__get_node_types_yaml_file_path(model_name, path)
+
+        is_update = True if action == self.ACTION_UPGRADE else False
+
+        self.__node_type_client.import_all(node_types_yaml_path, payload_json_str, is_update)
+
+    def __get_node_types_metadata_file_path(self, model_name, node_types_folder):
+        metadata_json_path = node_types_folder / 'metadata.json'
+        if not os.path.isfile(metadata_json_path):
+            error_msg = "Missing metadata.json file for model '{}'. Expected path '{}'".format(model_name,
+                                                                                               metadata_json_path)
+            raise Exception(error_msg)
+        return metadata_json_path
+
+    def __get_node_types_yaml_file_path(self, model_name, node_types_folder):
+        node_types_yaml_path = node_types_folder / self.NODE_TYPE_FILE
+        if not os.path.isfile(node_types_yaml_path):
+            error_msg = "Missing {} file for model '{}'. Expected path '{}'".format(self.NODE_TYPE_FILE, model_name,
+                                                                                    node_types_yaml_path)
+            raise Exception(error_msg)
+        return node_types_yaml_path
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py
new file mode 100644
index 0000000000..7937002567
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/node_type_client.py
@@ -0,0 +1,75 @@
+#  -
+#   ============LICENSE_START=======================================================
+#   Copyright (C) 2021 Nordix Foundation.
+#   ================================================================================
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#   SPDX-License-Identifier: Apache-2.0
+#   ============LICENSE_END=========================================================
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  ============LICENSE_END========================================================
+
+from io import BytesIO
+from pathlib import Path
+
+import pycurl
+
+from sdcBePy.common import logger
+
+
+class NodeTypeClient:
+
+    def __init__(self, sdc_be_proxy):
+        self.__base_path = Path('/sdc2/rest/v1/catalog/upload')
+        self.__import_all_path = self.__base_path / 'resource/import'
+        self.__sdc_be_proxy = sdc_be_proxy
+
+    def import_all(self, node_type_yaml_path, node_type_metadata_json_str, is_update=False):
+        logger.debug("Starting to import node types '{}'".format(node_type_yaml_path))
+
+        multi_part_form_data = []
+
+        node_type_yaml_param = ('nodeTypesYaml', (pycurl.FORM_FILE, str(node_type_yaml_path)))
+        multi_part_form_data.append(node_type_yaml_param)
+
+        node_type_metadata_json_param = ('nodeTypeMetadataJson', (
+            pycurl.FORM_CONTENTS, node_type_metadata_json_str,
+            pycurl.FORM_CONTENTTYPE, 'application/json'
+        ))
+        multi_part_form_data.append(node_type_metadata_json_param)
+
+        if is_update is not None:
+            create_new_version_param = ('createNewVersion', (
+                pycurl.FORM_CONTENTS, str(is_update).lower(),
+                pycurl.FORM_CONTENTTYPE, 'text/plain'
+            ))
+            multi_part_form_data.append(create_new_version_param)
+
+        response_buffer = BytesIO()
+        response_code = self.__sdc_be_proxy.post_file(str(self.__import_all_path), multi_part_form_data,
+                                                      response_buffer)
+        logger.debug("Import all node types response code '{}'".format(response_code))
+        if response_code != 201:
+            error_msg = "Failed to import node types '{}'".format(node_type_yaml_path)
+            logger.log(error_msg, response_buffer.getvalue())
+            raise Exception(error_msg)
+        logger.log("Failed to import node types '{}'".format(node_type_yaml_path))
+
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
index 663d27b435..6af44920ca 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
@@ -9,6 +9,7 @@ from sdcBePy.common.normative.main import process_element_list, process_type_lis
 from sdcBePy.tosca.main import parse_and_create_proxy
 from sdcBePy.tosca.models.model_client import ModelClient
 from sdcBePy.tosca.models.model_import_manager import ModelImportManager
+from sdcBePy.tosca.models.node_type_client import NodeTypeClient
 from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
     get_normative_element_with_metadata_list
 from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \
@@ -25,7 +26,8 @@ def main(sdc_be_proxy):
     base_file_location = os.getcwd() + "/"
     logger.debug("working directory =" + base_file_location)
 
-    model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+    model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy),
+                                              NodeTypeClient(sdc_be_proxy))
     try:
         model_import_manager.deploy_models()
     except Exception as ex: