Update backend-init to use new endpoints allowing specific model

Issue-ID: SDC-3676
Signed-off-by: davsad <david.sadlier@est.tech>
Change-Id: I5e77185dcaa0f3172958ac93198ae2df2f17366b

5 files changed, 69 insertions, 21 deletions

diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py
index ad8338cee4..805f0b0167 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/main.py
@@ -11,14 +11,16 @@ from sdcBePy import properties
 colors = BColors()
 
 
-def process_element_list(normative_elements_list, sdc_be_proxy):
+def process_element_list(normative_elements_list, sdc_be_proxy, model=None):
     attempt = 0
     for normative_element in normative_elements_list:
+        if normative_element is None:
+            continue
         while True:
             attempt += 1
             try:
                 process_and_create_normative_element(normative_element,
-                                                     sdc_be_proxy=sdc_be_proxy)
+                                                     sdc_be_proxy=sdc_be_proxy, model=model)
                 break
             except ResourceCreationError as e:
                 _check_and_retry(attempt, e.error_code, e.message)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
index 654a755863..8506207ff9 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
@@ -10,7 +10,7 @@ from sdcBePy.common.errors import ResourceCreationError
 
 def process_and_create_normative_element(normative_element,
                                          scheme=None, be_host=None, be_port=None, header=None, admin_user=None, sdc_be_proxy=None,
-                                         debug=False,
+                                         model=None, debug=False,
                                          exit_on_success=False):
     if sdc_be_proxy is None:
         sdc_be_proxy = SdcBeProxy(be_host, be_port, header, scheme, admin_user, debug=debug)
@@ -21,31 +21,33 @@ def process_and_create_normative_element(normative_element,
                               url_suffix,
                               element_name,
                               element_from_name,
+                              model,
                               with_metadata,
                               exit_on_success)
 
 
 def _create_normative_element(sdc_be_proxy, file_dir,
-                              url_suffix, element_name, element_form_name, with_metadata=False,
+                              url_suffix, element_name, element_form_name, model, with_metadata=False,
                               exit_on_success=False):
     result = _send_request(sdc_be_proxy,
                            file_dir,
                            url_suffix,
                            element_name,
                            element_form_name,
+                           model,
                            with_metadata)
     print_and_check_result(result, exit_on_success)
 
 
 def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name,
-                  element_form_name,
+                  element_form_name, model,
                   with_metadata=False):
     try:
         log("create normative element ", element_name)
 
         type_file_name = file_dir + element_name
         multi_part_form_data = _create_multipart_form_data(element_form_name, type_file_name, with_metadata,
-                                                           element_name)
+                                                           element_name, model)
 
         debug("http request url =", url_suffix)
         http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data)
@@ -62,12 +64,15 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name,
         return element_name, None, None
 
 
-def _create_multipart_form_data(element_form_name, type_file_name, with_metadata, element_name):
+def _create_multipart_form_data(element_form_name, type_file_name, with_metadata, element_name, model):
     tosca_type_zip_part = _create_zip_file_multi_part(element_form_name, type_file_name, element_name)
     multi_part_form_data = [tosca_type_zip_part]
     if with_metadata:
         metadata_type_part = _create_metadata_multipart(type_file_name)
         multi_part_form_data.append(metadata_type_part)
+    if model is not None:
+        model_data = ("model", model)
+        multi_part_form_data.append(model_data)
     debug(multi_part_form_data)
     return multi_part_form_data
 
@@ -76,7 +81,6 @@ def _create_metadata_multipart(type_file_name):
     metadata = _create_json_metadata_str(type_file_name)
     return "toscaTypeMetadata", metadata
 
-
 def _create_zip_file_multi_part(element_form_name, type_file_name, element_name):
     zf = zipfile.ZipFile(type_file_name + ".zip", "w")
     zf.write(type_file_name + '.yml', element_name + '.yml')
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
index 29b01bbdee..fd1cecd429 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
@@ -22,7 +22,8 @@ from pathlib import Path
 import pycurl
 
 from sdcBePy.common import logger
-
+from sdcBePy.common.normative.main import process_element_list, process_type_list
+from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, get_normative_element_with_metadata_list
 
 class ModelClient:
 
@@ -76,6 +77,21 @@ class ModelClient:
             raise Exception(error_msg)
         logger.log("Updated model", model_name)
 
+    def import_model_elements(self, model_payload_dict, tosca_elements_import_path, with_metadata=False):
+        model_name = model_payload_dict['name']
+        logger.debug("Starting import of normative elements for model '{}'".format(model_name))
+        if with_metadata:
+            process_element_list(get_normative_element_with_metadata_list(tosca_elements_import_path), self.__sdc_be_proxy, model=model_name)
+        else:
+            process_element_list(get_normative_element_candidate_list(tosca_elements_import_path), self.__sdc_be_proxy, model=model_name)
+        logger.log("Finished importing normative elements for model", model_name)
+
+    def import_model_types(self, model_payload_dict, types_list, upgrade):
+        model_name = model_payload_dict['name']
+        logger.debug("Starting import of normative types for model '{}'".format(model_name))
+        process_type_list(types_list, self.__sdc_be_proxy, upgrade)
+        logger.log("Finished importing normative types for model", model_name)
+
     @staticmethod
     def __parse_to_json_str(model_payload_dict):
         return json.dumps(model_payload_dict)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
index 016de03b29..51c6e4ea97 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
@@ -18,6 +18,7 @@
 import json
 import os
 import zipfile
+from sdcBePy.tosca.models.normativeTypeCandidate import NormativeTypeCandidate
 from pathlib import Path
 
 
@@ -29,6 +30,7 @@ class ModelImportManager:
     IMPORTS_FOLDER_NAME = 'imports'
     ACTION_UPGRADE = 'upgrade'
     ACTION_INIT = 'init'
+    NODE_TYPES = 'node-types/'
 
     def __init__(self, model_imports_path, model_client):
         self.__model_base_path = model_imports_path
@@ -41,12 +43,20 @@ class ModelImportManager:
             model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT)
             model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT)
             self.__model_client.create_model(model_payload_dict, model_imports_zip_path)
+            tosca_path = self.__get_model_tosca_path(self.ACTION_INIT, model_folder_name)
+            self.__model_client.import_model_elements(model_payload_dict, tosca_path)
+            if os.path.isdir(tosca_path + self.NODE_TYPES):
+                self.__model_client.import_model_types(model_payload_dict, self.__get_model_normative_type_candidate(tosca_path), False)
+            self.__model_client.import_model_elements(model_payload_dict, tosca_path, True)
 
     def update_models(self):
         for model_folder_name in self.__get_model_upgrade_list():
             model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE)
             model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE)
             self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path)
+            tosca_path = self.__get_model_tosca_path(self.ACTION_UPGRADE, model_folder_name)
+            if os.path.isdir(tosca_path + self.NODE_TYPES):
+                self.__model_client.import_model_types(model_payload_dict, self.__get_model_normative_type_candidate(tosca_path), True)
 
     def __get_model_init_list(self):
         return self.__get_model_list(self.__model_init_path)
@@ -62,6 +72,10 @@ class ModelImportManager:
             break
         return model_list
 
+    def __get_model_normative_type_candidate(self, tosca_path):
+        path = tosca_path + self.NODE_TYPES
+        return [NormativeTypeCandidate(path, self.__read_model_type_json(path))]
+
     def __zip_model_imports(self, model, action_type) -> Path:
         base_path = self.__get_base_action_path(action_type)
         model_path = base_path / model
@@ -81,6 +95,13 @@ class ModelImportManager:
         json_data = json.load(json_file, strict=False)
         return json.dumps(json_data)
 
+    def __read_model_type_json(self, tosca_path):
+        path = tosca_path + "types.json"
+        if not os.path.isfile(path):
+            return []
+        json_file = open(path)
+        return json.load(json_file)
+
     def __read_model_payload(self, model, action_type) -> dict:
         base_path = self.__get_base_action_path(action_type)
         model_payload_path = base_path / model / "payload.json"
@@ -89,3 +110,6 @@ class ModelImportManager:
 
     def __get_base_action_path(self, action_type) -> Path:
         return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path
+
+    def __get_model_tosca_path(self, action, model):
+        return str(self.__get_base_action_path(action) / model) + "/tosca/"
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/normativeElementsList.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/normativeElementsList.py
index cfef95d427..ffd412f38f 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/normativeElementsList.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/normativeElementsList.py
@@ -1,6 +1,6 @@
+from os import path
 from sdcBePy.tosca.models.normativeElementCandidate import NormativeElementCandidate
 
-
 def get_normative_element_candidate_list(base_file_location):
     return [
         get_data(base_file_location),
@@ -10,51 +10,53 @@ def get_normative_element_candidate_list(base_file_location):
         get_categories(base_file_location)
     ]
 
-
 def get_normative_element_with_metadata_list(base_file_location):
     return [
         get_group(base_file_location),
         get_policy(base_file_location)
     ]
 
+def get_normative_candidate(base_file_location, url, filename, zip_name, with_metadata=False):
+    if path.isdir(base_file_location):
+        return NormativeElementCandidate(base_file_location, url, filename, zip_name, with_metadata=with_metadata)
 
 def get_data(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "data-types/",
+    return get_normative_candidate(base_file_location + "data-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/datatypes",
                                      "dataTypes",
                                      "dataTypesZip")
 
 
 def get_capability(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "capability-types/",
+    return get_normative_candidate(base_file_location + "capability-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/capability",
                                      "capabilityTypes",
                                      "capabilityTypeZip")
 
 
 def get_relationship(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "relationship-types/",
+    return get_normative_candidate(base_file_location + "relationship-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/relationship",
                                      "relationshipTypes",
                                      "relationshipTypeZip")
 
 
 def get_interface_lifecycle(base_file_location="../../../import/tosca/"):
-    return NormativeElementCandidate(base_file_location + "interface-lifecycle-types/",
+    return get_normative_candidate(base_file_location + "interface-lifecycle-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/interfaceLifecycle",
                                      "interfaceLifecycleTypes",
                                      "interfaceLifecycleTypeZip")
 
 
 def get_categories(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "categories/",
+    return get_normative_candidate(base_file_location + "categories/",
                                      "/sdc2/rest/v1/catalog/uploadType/categories",
                                      "categoryTypes",
                                      "categoriesZip")
 
 
 def get_group(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "group-types/",
+    return get_normative_candidate(base_file_location + "group-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/grouptypes",
                                      "groupTypes",
                                      "groupTypesZip",
@@ -62,7 +64,7 @@ def get_group(base_file_location="/"):
 
 
 def get_policy(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "policy-types/",
+    return get_normative_candidate(base_file_location + "policy-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/policytypes",
                                      "policyTypes",
                                      "policyTypesZip",
@@ -70,7 +72,7 @@ def get_policy(base_file_location="/"):
 
 
 def get_annotation(base_file_location="/"):
-    return NormativeElementCandidate(base_file_location + "annotation-types",
+    return get_normative_candidate(base_file_location + "annotation-types/",
                                      "/sdc2/rest/v1/catalog/uploadType/annotationtypes",
-                                     "annotationTypesZip",
-                                     "annotationTypes")
+                                     "annotationTypes",
+                                     "annotationTypesZip")