Init ONAP model imports using the model API

Creates a client for the model endpoint in the catalog init scripts.
Introduces the directory structure to provide the models along its
imports, separated by init/upgrade phase.
Each model structure will be zipped and uploaded to the endpoint,
based on the model directory name.

Change-Id: I0392c1e6d3a29b30567b11016041a8e9cccbc745
Issue-ID: SDC-3615
Signed-off-by: André Schmid <andre.schmid@est.tech>

6 files changed, 245 insertions, 34 deletions

diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
index 5ef3173907..654a755863 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/normative/toscaElements.py
@@ -47,6 +47,7 @@ def _send_request(sdc_be_proxy, file_dir, url_suffix, element_name,
         multi_part_form_data = _create_multipart_form_data(element_form_name, type_file_name, with_metadata,
                                                            element_name)
 
+        debug("http request url =", url_suffix)
         http_res = sdc_be_proxy.post_file(url_suffix, multi_part_form_data)
         if http_res is not None:
             debug("http response =", http_res)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
index 169979ae73..f2bef53564 100755
--- a/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/common/sdcBeProxy.py
@@ -21,7 +21,7 @@ class SdcBeProxy:
             raise AttributeError("The be_host, be_port, scheme or admin_user are missing")
         url = get_url(be_ip, be_port, scheme)
         self.con = connector if connector \
-            else CurlConnector(url, user_id, header, scheme=scheme, debug=debug)
+            else CurlConnector(url, user_id, header, protocol=scheme, debug=debug)
 
     def check_backend(self):
         return self.con.get('/sdc2/rest/v1/user/jh0003')
@@ -52,8 +52,11 @@ class SdcBeProxy:
     def get_normatives(self):
         return self.con.get("/sdc2/rest/v1/screen", with_buffer=True)
 
-    def post_file(self, path, multi_part_form_data):
-        return self.con.post_file(path, multi_part_form_data)
+    def post_file(self, path, multi_part_form_data, buffer=None):
+        return self.con.post_file(path, multi_part_form_data, buffer)
+
+    def put_file(self, path, multi_part_form_data, buffer=None):
+        return self.con.put_file(path, multi_part_form_data, buffer)
 
     def get_response_from_buffer(self):
         value = self.con.buffer.getvalue()
@@ -68,17 +71,13 @@ class CurlConnector:
     CONTENT_TYPE_HEADER = "Content-Type: application/json"
     ACCEPT_HEADER = "Accept: application/json; charset=UTF-8"
 
-    def __init__(self, url, user_id_header, header, buffer=None, scheme="http", debug=False):
-        self.c = pycurl.Curl()
-        self.c.setopt(pycurl.HEADER, True)
+    def __init__(self, url, user_id_header, header, buffer=None, protocol="http", debug=False):
+        self.__debug = debug
+        self.__protocol = protocol
+        self.c = self.__build_default_curl()
 
         self.user_header = "USER_ID: " + user_id_header
-
-        if not debug:
-            # disable printing not necessary logs in the terminal
-            self.c.setopt(pycurl.WRITEFUNCTION, lambda x: None)
-        else:
-            self.c.setopt(pycurl.VERBOSE, 1)
+        self.url = url
 
         if not buffer:
             self.buffer = BytesIO()
@@ -88,9 +87,6 @@ class CurlConnector:
         else:
             self.basicauth_header = "Authorization: Basic " + header
 
-        self.url = url
-        self._check_schema(scheme)
-
     def get(self, path, buffer=None, with_buffer=False):
         try:
             self.c.setopt(pycurl.URL, self.url + path)
@@ -99,7 +95,6 @@ class CurlConnector:
                                               CurlConnector.ACCEPT_HEADER,
                                               self.basicauth_header])
 
-
             if with_buffer:
                 write = self.buffer.write if not buffer else buffer.write
                 self.c.setopt(pycurl.WRITEFUNCTION, write)
@@ -115,9 +110,9 @@ class CurlConnector:
             self.c.setopt(pycurl.POST, 1)
 
             self.c.setopt(pycurl.HTTPHEADER, [self.user_header,
-                                           CurlConnector.CONTENT_TYPE_HEADER,
-                                           CurlConnector.ACCEPT_HEADER,
-                                           self.basicauth_header])
+                                              CurlConnector.CONTENT_TYPE_HEADER,
+                                              CurlConnector.ACCEPT_HEADER,
+                                              self.basicauth_header])
 
             self.c.setopt(pycurl.POSTFIELDS, data)
 
@@ -132,9 +127,7 @@ class CurlConnector:
         try:
             self.c.setopt(pycurl.URL, self.url + path)
             self.c.setopt(pycurl.POST, 1)
-            self.c.setopt(pycurl.HTTPHEADER, [self.user_header,
-                                           self.basicauth_header])
-
+            self.c.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header])
 
             self.c.setopt(pycurl.HTTPPOST, post_body)
 
@@ -143,15 +136,40 @@ class CurlConnector:
 
             self.c.perform()
             self.c.setopt(pycurl.POST, 0)
-
             return self.c.getinfo(pycurl.RESPONSE_CODE)
-        except pycurl.error:
+        except pycurl.error as ex:
+            print(ex)
             return 111
 
-    def _check_schema(self, scheme):
-        if scheme == 'https':
-            self.c.setopt(pycurl.SSL_VERIFYPEER, 0)
-            self.c.setopt(pycurl.SSL_VERIFYHOST, 0)
+    def put_file(self, path, post_body, response_write_buffer=None):
+        curl = self.__build_default_curl()
+        curl.setopt(pycurl.URL, self.url + path)
+        curl.setopt(pycurl.HTTPHEADER, [self.user_header, self.basicauth_header])
+        curl.setopt(pycurl.CUSTOMREQUEST, "PUT")
+
+        curl.setopt(pycurl.HTTPPOST, post_body)
+
+        write = self.buffer.write if not response_write_buffer else response_write_buffer.write
+        curl.setopt(pycurl.WRITEFUNCTION, write)
+
+        curl.perform()
+        response_code = curl.getinfo(pycurl.RESPONSE_CODE)
+        curl.close()
+        return response_code
+
+    def __build_default_curl(self):
+        curl = pycurl.Curl()
+        if not self.__debug:
+            # disable printing not necessary logs in the terminal
+            curl.setopt(pycurl.WRITEFUNCTION, lambda x: None)
+        else:
+            curl.setopt(pycurl.VERBOSE, 1)
+
+        if self.__protocol == 'https':
+            curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
+        curl.setopt(pycurl.HEADER, True)
+        return curl
 
     def __del__(self):
         self.c.close()
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
index 9ac820071a..067f110efc 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/imports/run.py
@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
 
 import os
+from pathlib import Path
 
 import sdcBePy.common.logger as logger
 from sdcBePy.common.normative.main import process_element_list, process_type_list
 from sdcBePy.tosca.main import parse_and_create_proxy
-from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
-    get_normative_element_with_metadata_list
-from sdcBePy.tosca.models.normativeTypesList import get_normative_type_candidate_list
+from sdcBePy.tosca.models import normativeElementsList
+from sdcBePy.tosca.models import normativeTypesList
+from sdcBePy.tosca.models.model_client import ModelClient
+from sdcBePy.tosca.models.model_import_manager import ModelImportManager
 
 
 def main(sdc_be_proxy, update_version):
@@ -16,9 +18,16 @@ def main(sdc_be_proxy, update_version):
     base_file_location = os.getcwd() + os.path.sep
     logger.debug("working directory =" + base_file_location)
 
-    process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
-    process_type_list(get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
-    process_element_list(get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
+    model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+    try:
+        model_import_manager.create_models()
+    except Exception as ex:
+        logger.log("An error has occurred while uploading the models: ", str(ex))
+        raise ex
+
+    process_element_list(normativeElementsList.get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
+    process_type_list(normativeTypesList.get_normative_type_candidate_list(base_file_location), sdc_be_proxy, update_version)
+    process_element_list(normativeElementsList.get_normative_element_with_metadata_list(base_file_location), sdc_be_proxy)
 
     logger.log("Script end ->", "All normatives imported successfully!")
     logger.print_and_exit(0, None)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
new file mode 100644
index 0000000000..29b01bbdee
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_client.py
@@ -0,0 +1,81 @@
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2021 Nordix Foundation
+#  ===============================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  ============LICENSE_END========================================================
+
+import json
+from io import BytesIO
+from pathlib import Path
+
+import pycurl
+
+from sdcBePy.common import logger
+
+
+class ModelClient:
+
+    def __init__(self, sdc_be_proxy):
+        self.__base_path = Path('/sdc2/rest/v1/catalog/model')
+        self.__imports_path = self.__base_path / 'imports'
+        self.__sdc_be_proxy = sdc_be_proxy
+
+    def create_model(self, model_payload_dict, model_imports_zip_path):
+        model_name = model_payload_dict['name']
+
+        logger.debug("Starting to create model '{}', zip path '{}'".format(model_name, model_imports_zip_path))
+
+        multi_part_form_data = []
+
+        model_zip_param = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path)))
+        multi_part_form_data.append(model_zip_param)
+
+        json_payload = self.__parse_to_json_str(model_payload_dict)
+        model_param = ('model', (
+            pycurl.FORM_CONTENTS, json_payload,
+            pycurl.FORM_CONTENTTYPE, 'application/json'
+        ))
+        multi_part_form_data.append(model_param)
+
+        response_buffer = BytesIO()
+        response_code = self.__sdc_be_proxy.post_file(str(self.__base_path), multi_part_form_data, response_buffer)
+        logger.debug("Create model response code '{}'".format(response_code))
+        if response_code != 201:
+            error_msg = "Failed to create model '{}'".format(model_name)
+            logger.log(error_msg, response_buffer.getvalue())
+            raise Exception(error_msg)
+        logger.log("Created model", model_name)
+
+    def update_model_imports(self, model_payload_dict, model_imports_zip_path):
+        model_name = model_payload_dict['name']
+        logger.debug("Starting to update model '{}', zip path '{}'".format(model_name, model_imports_zip_path))
+
+        multi_part_form_data = []
+
+        model_zip_post = ('modelImportsZip', (pycurl.FORM_FILE, str(model_imports_zip_path)))
+        multi_part_form_data.append(('modelName', model_name))
+        multi_part_form_data.append(model_zip_post)
+
+        response_buffer = BytesIO()
+        response_code = self.__sdc_be_proxy.put_file(str(self.__imports_path), multi_part_form_data, response_buffer)
+        logger.debug("Update model response code '{}'".format(response_code))
+        if response_code != 204:
+            error_msg = "Failed to update model '{}'".format(model_name)
+            logger.log(error_msg, response_buffer.getvalue())
+            raise Exception(error_msg)
+        logger.log("Updated model", model_name)
+
+    @staticmethod
+    def __parse_to_json_str(model_payload_dict):
+        return json.dumps(model_payload_dict)
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
new file mode 100644
index 0000000000..016de03b29
--- /dev/null
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/models/model_import_manager.py
@@ -0,0 +1,91 @@
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2021 Nordix Foundation
+#  ===============================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+#  ============LICENSE_END========================================================
+
+import json
+import os
+import zipfile
+from pathlib import Path
+
+
+class ModelImportManager:
+    """Manages the model imports directory"""
+
+    INIT_FOLDER_NAME = 'init'
+    UPGRADE_FOLDER_NAME = 'upgrade'
+    IMPORTS_FOLDER_NAME = 'imports'
+    ACTION_UPGRADE = 'upgrade'
+    ACTION_INIT = 'init'
+
+    def __init__(self, model_imports_path, model_client):
+        self.__model_base_path = model_imports_path
+        self.__model_init_path = self.__model_base_path / self.INIT_FOLDER_NAME
+        self.__model_upgrade_path = self.__model_base_path / self.UPGRADE_FOLDER_NAME
+        self.__model_client = model_client
+
+    def create_models(self):
+        for model_folder_name in self.__get_model_init_list():
+            model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_INIT)
+            model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_INIT)
+            self.__model_client.create_model(model_payload_dict, model_imports_zip_path)
+
+    def update_models(self):
+        for model_folder_name in self.__get_model_upgrade_list():
+            model_imports_zip_path = self.__zip_model_imports(model_folder_name, self.ACTION_UPGRADE)
+            model_payload_dict = self.__read_model_payload(model_folder_name, self.ACTION_UPGRADE)
+            self.__model_client.update_model_imports(model_payload_dict, model_imports_zip_path)
+
+    def __get_model_init_list(self):
+        return self.__get_model_list(self.__model_init_path)
+
+    def __get_model_upgrade_list(self):
+        return self.__get_model_list(self.__model_upgrade_path)
+
+    @staticmethod
+    def __get_model_list(path):
+        model_list = []
+        for (dirpath, dirnames, filenames) in os.walk(path):
+            model_list.extend(dirnames)
+            break
+        return model_list
+
+    def __zip_model_imports(self, model, action_type) -> Path:
+        base_path = self.__get_base_action_path(action_type)
+        model_path = base_path / model
+        model_imports_path = base_path / model / self.IMPORTS_FOLDER_NAME
+        zip_file_path = model_path / "{}.zip".format(model)
+        zip_file = zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED)
+        for root, dirs, files in os.walk(model_imports_path):
+            for file in files:
+                zip_file.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), model_imports_path))
+        zip_file.close()
+        return zip_file_path
+
+    def __read_model_payload_as_string(self, model, action_type) -> str:
+        base_path = self.__get_base_action_path(action_type)
+        model_payload_path = base_path / model / "payload.json"
+        json_file = open(model_payload_path)
+        json_data = json.load(json_file, strict=False)
+        return json.dumps(json_data)
+
+    def __read_model_payload(self, model, action_type) -> dict:
+        base_path = self.__get_base_action_path(action_type)
+        model_payload_path = base_path / model / "payload.json"
+        json_file = open(model_payload_path)
+        return json.load(json_file, strict=False)
+
+    def __get_base_action_path(self, action_type) -> Path:
+        return self.__model_init_path if action_type == self.INIT_FOLDER_NAME else self.__model_upgrade_path
diff --git a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
index cef7dd74db..44a25a0136 100644
--- a/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
+++ b/catalog-be/src/main/resources/scripts/sdcBePy/tosca/upgrade/run.py
@@ -1,11 +1,14 @@
 #!/usr/bin/env python3
 
 import os
+from pathlib import Path
 
 from sdcBePy.common import logger
 from sdcBePy.common.logger import print_and_exit
 from sdcBePy.common.normative.main import process_element_list, process_type_list
 from sdcBePy.tosca.main import parse_and_create_proxy
+from sdcBePy.tosca.models.model_client import ModelClient
+from sdcBePy.tosca.models.model_import_manager import ModelImportManager
 from sdcBePy.tosca.models.normativeElementsList import get_normative_element_candidate_list, \
     get_normative_element_with_metadata_list
 from sdcBePy.tosca.models.normativeToUpdateList import TypesToUpdate, get_heat_and_normative_to_update_list, \
@@ -21,6 +24,14 @@ def main(sdc_be_proxy):
     # base_file_location = os.getcwd() + "/../../../../import/tosca/"
     base_file_location = os.getcwd() + "/"
     logger.debug("working directory =" + base_file_location)
+
+    model_import_manager = ModelImportManager(Path(base_file_location) / 'models', ModelClient(sdc_be_proxy))
+    try:
+        model_import_manager.update_models()
+    except Exception as ex:
+        logger.log("An error has occurred while uploading the models: ", str(ex))
+        raise ex
+
     process_element_list(get_normative_element_candidate_list(base_file_location), sdc_be_proxy)
 
     all_types = get_all_types()