Merge "Add Artifact Manager service."

5 files changed, 630 insertions, 0 deletions

diff --git a/ms/artifact-manager/manager/__init__.py b/ms/artifact-manager/manager/__init__.py
new file mode 100644
index 000000000..21236908e
--- /dev/null
+++ b/ms/artifact-manager/manager/__init__.py
@@ -0,0 +1,14 @@
+"""Copyright 2019 Deutsche Telekom.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
diff --git a/ms/artifact-manager/manager/configuration.py b/ms/artifact-manager/manager/configuration.py
new file mode 100644
index 000000000..0af2c22cc
--- /dev/null
+++ b/ms/artifact-manager/manager/configuration.py
@@ -0,0 +1,139 @@
+"""Copyright 2019 Deutsche Telekom.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import logging
+import os
+from configparser import ConfigParser, SectionProxy
+from distutils.util import strtobool
+from logging import Logger
+from pathlib import Path, PurePath
+from typing import NoReturn
+
+from onaplogging import monkey
+from onaplogging.mdcformatter import MDCFormatter  # noqa
+
+monkey.patch_loggingYaml()
+
+
+DEFAUL_CONFIGURATION_FILE: str = str(PurePath(Path().absolute(), "../configuration.ini"))
+SUPPLIED_CONFIGURATION_FILE: str = os.environ.get("CONFIGURATION")
+CONFIGURATION_FILE: str = str(os.path.expanduser(Path(SUPPLIED_CONFIGURATION_FILE or DEFAUL_CONFIGURATION_FILE)))
+
+
+class ArtifactManagerConfiguration:
+    """ServerConfiguration class loads configuration from config INI files."""
+
+    def __init__(self, config_file_path: str) -> NoReturn:
+        """Initialize configuration class instance.
+
+        Configuration is loaded from file provided as a parameter. Environment variables are loaded also.
+        Logger for object is created with the name same as the class name.
+        :param config_file_path: Path to configuration file.
+        """
+        self.config_file_path = config_file_path
+        self.config = ConfigParser(os.environ)
+        self.config.read(config_file_path, encoding="utf-8")
+
+    @property
+    def configuration_directory(self) -> str:
+        """Get directory path to a directory with configuration ini file.
+
+        This is used to handle relative file paths in config file.
+        """
+        return os.path.dirname(self.config_file_path)
+
+    def get_section(self, section_name: str) -> SectionProxy:
+        """Get the section from configuration file.
+
+        :param section_name: Name of the section to get
+        :raises: KeyError
+        :return: SectionProxy object for given section name
+        """
+        return self.config[section_name]
+
+    def __getitem__(self, key: str) -> SectionProxy:
+        """Get the section from configuration file.
+
+        This method just calls the get_section method but allows us to use it as key lookup
+
+        :param section_name: Name of the section to get
+        :raises: KeyError
+        :return: SectionProxy object for given section name
+        """
+        return self.get_section(key)
+
+    def get_property(self, section_name: str, property_name: str) -> str:
+        """Get the property value from *section_name* section.
+
+        :param section_name: Name of the section config file section on which property is set
+        :param property_name: Name of the property to get
+        :raises: configparser.NoOptionError
+        :return: String value of the property
+        """
+        return self.config.get(section_name, property_name)
+
+    def artifact_manager_property(self, property_name: str) -> str:
+        """Get the property value from *artifactManagerServer* section.
+
+        :param property_name: Name of the property to get
+        :raises: configparser.NoOptionError
+        :return: String value of the property
+        """
+        return self.config.get("artifactManagerServer", property_name)
+
+
+config = ArtifactManagerConfiguration(CONFIGURATION_FILE)
+
+
+def prepare_logger(log_file_path: str, development_mode: bool, config: ArtifactManagerConfiguration) -> callable:
+    """Base MDC logger configuration.
+
+    Level depends on the *development_mode* flag: DEBUG if development_mode is set or INFO otherwise.
+    Console handler is created from MDC settings from onappylog library.
+
+    :param log_file_path: Path to the log file, where logs are going to be saved.
+    :param development_mode: Boolean type value which means if logger should be setup in development mode or not
+    :param config: Configuration class so we can fetch app settings (paths) to logger.
+    :return: callable
+    """
+    logging_level: int = logging.DEBUG if development_mode else logging.INFO
+    logging.basicConfig(filename=log_file_path, level=logging_level)
+    logging.config.yamlConfig(
+        filepath=Path(config.configuration_directory, config["artifactManagerServer"]["logConfig"])
+    )
+
+    console: logging.StreamHandler = logging.StreamHandler()
+    console.setLevel(logging_level)
+    formatter: logging.Formatter = MDCFormatter(
+        fmt="%(asctime)s:[%(name)s] %(created)f %(module)s %(funcName)s %(pathname)s %(process)d %(levelno)s :[ %(threadName)s  %(thread)d]: [%(mdc)s]: [%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s",  # noqa
+        mdcfmt="{RequestID} {InvocationID} {ServiceName} {PartnerName} {BeginTimestamp} {EndTimestamp} {ElapsedTime} {StatusCode} {TargetEntity} {TargetServiceName} {Server}",  # noqa
+        # Important: We cannot use %f here because this datetime format is used by time library, not datetime.
+        datefmt="%Y-%m-%dT%H:%M:%S%z",
+    )
+    console.setFormatter(formatter)
+
+    def get_logger(name: str) -> Logger:
+        """Get a new logger with predefined MDC handler."""
+        logger: Logger = logging.getLogger(name)
+        logger.addHandler(console)
+        return logger
+
+    return get_logger
+
+
+get_logger = prepare_logger(
+    config.artifact_manager_property("logFile"),
+    strtobool(config["artifactManagerServer"].get("debug", "false")),
+    config,
+)
diff --git a/ms/artifact-manager/manager/errors.py b/ms/artifact-manager/manager/errors.py
new file mode 100644
index 000000000..feafd7668
--- /dev/null
+++ b/ms/artifact-manager/manager/errors.py
@@ -0,0 +1,64 @@
+"""Copyright 2019 Deutsche Telekom.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+class ArtifactManagerError(Exception):
+    """Base Artifact Manager exception class."""
+
+    status_code: int = 0
+    message: str = "Error"
+
+    def __init__(self, message: str = None) -> None:
+        """Initialize exception with optional message."""
+        if message:
+            self.message: str = message
+
+    @property
+    def status_code(self) -> int:
+        """Artifact Manager error class status code.
+
+        Base class has no and shouldn't have any status code.
+        """
+        if self.status_code == 0:
+            raise NotImplementedError
+        return self.status_code
+
+
+class InvalidRequestError(ArtifactManagerError):
+    """Raised when request has invalid or incomplete data."""
+
+    status_code: int = 500
+    message: str = "Invalid request"
+
+
+class ArtifactNotFoundError(ArtifactManagerError):
+    """Raised when requested artifact doesn't exist in system."""
+
+    status_code: int = 500
+    message: str = "Artifact not found"
+
+
+class ArtifactIOError(ArtifactManagerError):
+    """Raised on input/output error."""
+
+    status_code: int = 500
+    message: str = "Artifact is corrupted"
+
+
+class ArtifactOverwriteError(ArtifactManagerError):
+    """Raised when we cannot remove old artifact to save new."""
+
+    status_code: int = 500
+    message: str = "Artifact already exists and cannot be overwritten"
diff --git a/ms/artifact-manager/manager/servicer.py b/ms/artifact-manager/manager/servicer.py
new file mode 100644
index 000000000..be740b0e3
--- /dev/null
+++ b/ms/artifact-manager/manager/servicer.py
@@ -0,0 +1,237 @@
+"""Copyright 2019 Deutsche Telekom.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import socket
+from datetime import datetime, timezone
+from functools import wraps
+from logging import Logger
+from typing import NoReturn, Union
+
+from grpc import ServicerContext
+from manager.configuration import get_logger
+from manager.errors import ArtifactManagerError, InvalidRequestError
+from manager.utils import Repository, RepositoryStrategy
+from onaplogging.mdcContext import MDC
+from proto.BluePrintManagement_pb2 import (
+    BluePrintDownloadInput,
+    BluePrintManagementOutput,
+    BluePrintRemoveInput,
+    BluePrintUploadInput,
+)
+from proto.BluePrintManagement_pb2_grpc import BluePrintManagementServiceServicer
+
+MDC_DATETIME_FORMAT = r"%Y-%m-%dT%H:%M:%S.%f%z"
+COMMON_HEADER_DATETIME_FORMAT = r"%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+def fill_common_header(func):
+    """Decorator to fill handler's output values which is the same type for each handler.
+
+    It copies commonHeader from request object and set timestamp value.
+
+    :param func: Handler function
+    :return: _handler decorator callable object
+    """
+
+    @wraps(func)
+    def _decorator(
+        servicer: "ArtifactManagerServicer",
+        request: Union[BluePrintDownloadInput, BluePrintRemoveInput, BluePrintUploadInput],
+        context: ServicerContext,
+    ) -> BluePrintManagementOutput:
+
+        if not all([request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion]):
+            raise InvalidRequestError("Request has to have set both BluePrint name and version")
+        output: BluePrintManagementOutput = func(servicer, request, context)
+        # Set same values for every handler
+        output.commonHeader.CopyFrom(request.commonHeader)
+        output.commonHeader.timestamp = datetime.utcnow().strftime(COMMON_HEADER_DATETIME_FORMAT)
+        return output
+
+    return _decorator
+
+
+def translate_exception_to_response(func):
+    """Decorator that translates Artifact Manager exceptions into proper responses.
+
+    :param func: Handler function
+    :return: _handler decorator callable object
+    """
+
+    @wraps(func)
+    def _handler(
+        servicer: "ArtifactManagerServicer",
+        request: Union[BluePrintDownloadInput, BluePrintRemoveInput, BluePrintUploadInput],
+        context: ServicerContext,
+    ) -> BluePrintManagementOutput:
+        try:
+            output: BluePrintManagementOutput = func(servicer, request, context)
+            output.status.code = 200
+            output.status.message = "success"
+        except ArtifactManagerError as error:
+            # If ArtifactManagerError is raises one of defined error occurs.
+            # Every ArtifactManagerError based exception has status_code paramenter
+            # which has to be set in output. Use also exception's message to
+            # set errorMessage of the output.
+            output: BluePrintManagementOutput = BluePrintManagementOutput()
+            output.status.code = error.status_code
+            output.status.message = "failure"
+            output.status.errorMessage = str(error.message)
+
+            servicer.fill_MDC_timestamps()
+            servicer.logger.error(
+                "Error while processing the message - blueprintName={} blueprintVersion={}".format(
+                    request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+                ),
+                extra={"mdc": MDC.result()},
+            )
+            MDC.clear()
+        return output
+
+    return _handler
+
+
+def prepare_logging_context(func):
+    """Decorator that prepares MDC logging context for logs inside the handler.
+
+    :param func: Handler function
+    :return: _handler decorator callable object
+    """
+
+    @wraps(func)
+    def _decorator(
+        servicer: "ArtifactManagerServicer",
+        request: Union[BluePrintDownloadInput, BluePrintRemoveInput, BluePrintUploadInput],
+        context: ServicerContext,
+    ) -> BluePrintManagementOutput:
+        MDC.put("RequestID", request.commonHeader.requestId)
+        MDC.put("InvocationID", request.commonHeader.subRequestId)
+        MDC.put("ServiceName", servicer.__class__.__name__)
+        MDC.put("PartnerName", request.commonHeader.originatorId)
+        started_at = datetime.utcnow().replace(tzinfo=timezone.utc)
+        MDC.put("BeginTimestamp", started_at.strftime(MDC_DATETIME_FORMAT))
+
+        # Adding processing_started_at to the servicer so later we'll have the data to calculate elapsed time.
+        servicer.processing_started_at = started_at
+
+        MDC.put("TargetEntity", "py-executor")
+        MDC.put("TargetServiceName", func.__name__)
+        MDC.put("Server", socket.getfqdn())
+
+        output: BluePrintManagementOutput = func(servicer, request, context)
+        MDC.clear()
+        return output
+
+    return _decorator
+
+
+class ArtifactManagerServicer(BluePrintManagementServiceServicer):
+    """ArtifactManagerServer class.
+
+    Implements methods defined in proto files to manage artifacts repository.
+    These methods are: download, upload and remove.
+    """
+
+    processing_started_at = None
+
+    def __init__(self) -> NoReturn:
+        """Instance of ArtifactManagerServer class initialization.
+
+        Create logger for class using class name and set configuration property.
+        """
+        self.logger: Logger = get_logger(self.__class__.__name__)
+        self.repository: Repository = RepositoryStrategy.get_reporitory()
+
+    def fill_MDC_timestamps(self, status_code: int = 200) -> NoReturn:
+        """Add MDC context timestamps "in place".
+
+        :param status_code: int with expected response status. Default: 200 (success)
+        """
+        now = datetime.utcnow().replace(tzinfo=timezone.utc)
+        MDC.put("EndTimestamp", now.strftime(MDC_DATETIME_FORMAT))
+
+        # Elapsed time measured in miliseconds
+        MDC.put("ElapsedTime", (now - self.processing_started_at).total_seconds() * 1000)
+
+        MDC.put("StatusCode", status_code)
+
+    @prepare_logging_context
+    @translate_exception_to_response
+    @fill_common_header
+    def downloadBlueprint(self, request: BluePrintDownloadInput, context: ServicerContext) -> BluePrintManagementOutput:
+        """Download blueprint file request method.
+
+        Currently it only logs when is called and all base class method.
+        :param request: BluePrintDownloadInput
+        :param context: ServicerContext
+        :return: BluePrintManagementOutput
+        """
+        output: BluePrintManagementOutput = BluePrintManagementOutput()
+        output.fileChunk.chunk = self.repository.download_blueprint(
+            request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+        )
+        self.fill_MDC_timestamps()
+        self.logger.info(
+            "Blueprint download successfuly processed - blueprintName={} blueprintVersion={}".format(
+                request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+            ),
+            extra={"mdc": MDC.result()},
+        )
+        return output
+
+    @prepare_logging_context
+    @translate_exception_to_response
+    @fill_common_header
+    def uploadBlueprint(self, request: BluePrintUploadInput, context: ServicerContext) -> BluePrintManagementOutput:
+        """Upload blueprint file request method.
+
+        Currently it only logs when is called and all base class method.
+        :param request: BluePrintUploadInput
+        :param context: ServicerContext
+        :return: BluePrintManagementOutput
+        """
+        self.repository.upload_blueprint(
+            request.fileChunk.chunk, request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+        )
+        self.fill_MDC_timestamps()
+        self.logger.info(
+            "Blueprint upload successfuly processed - blueprintName={} blueprintVersion={}".format(
+                request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+            ),
+            extra={"mdc": MDC.result()},
+        )
+        return BluePrintManagementOutput()
+
+    @prepare_logging_context
+    @translate_exception_to_response
+    @fill_common_header
+    def removeBlueprint(self, request: BluePrintRemoveInput, context: ServicerContext) -> BluePrintManagementOutput:
+        """Remove blueprint file request method.
+
+        Currently it only logs when is called and all base class method.
+        :param request: BluePrintRemoveInput
+        :param context: ServicerContext
+        :return: BluePrintManagementOutput
+        """
+        self.repository.remove_blueprint(
+            request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+        )
+        self.fill_MDC_timestamps()
+        self.logger.info(
+            "Blueprint removal successfuly processed - blueprintName={} blueprintVersion={}".format(
+                request.actionIdentifiers.blueprintName, request.actionIdentifiers.blueprintVersion
+            ),
+            extra={"mdc": MDC.result()},
+        )
+        return BluePrintManagementOutput()
diff --git a/ms/artifact-manager/manager/utils.py b/ms/artifact-manager/manager/utils.py
new file mode 100644
index 000000000..da4cd9f9d
--- /dev/null
+++ b/ms/artifact-manager/manager/utils.py
@@ -0,0 +1,176 @@
+"""Copyright 2019 Deutsche Telekom.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import shutil
+from abc import ABC, abstractmethod
+from io import BytesIO
+from pathlib import Path
+from zipfile import ZipFile, is_zipfile
+
+from manager.configuration import config
+from manager.errors import ArtifactNotFoundError, ArtifactOverwriteError, InvalidRequestError
+
+
+class Repository(ABC):
+    """Abstract repository class.
+
+    Defines repository methods.
+    """
+
+    @abstractmethod
+    def upload_blueprint(self, file: bytes, name: str, version: str) -> None:
+        """Store blueprint file in the repository.
+
+        :param file: File to save
+        :param name: Blueprint name
+        :param version: Blueprint version
+        """
+
+    @abstractmethod
+    def download_blueprint(self, name: str, version: str) -> bytes:
+        """Download blueprint file from repository.
+
+        :param name: Blueprint name
+        :param version: Blueprint version
+        :return: Zipped Blueprint file bytes
+        """
+
+    @abstractmethod
+    def remove_blueprint(self, name: str, version: str) -> None:
+        """Remove blueprint file from repository.
+
+        :param name: Blueprint name
+        :param version: Blueprint version
+        """
+
+
+class FileRepository(Repository):
+    """Store blueprints on local directory."""
+
+    base_path = None
+
+    def __init__(self, base_path: Path) -> None:
+        """Initialize the repository while passing the needed path.
+
+        :param base_path: Local OS path on which blueprint files reside.
+        """
+        self.base_path = base_path
+
+    def __remove_directory_tree(self, full_path: str) -> None:
+        """Remove specified path.
+
+        :param full_path: Full path to a directory.
+        :raises: FileNotFoundError
+        """
+        try:
+            shutil.rmtree(full_path, ignore_errors=False)
+        except OSError:
+            raise ArtifactNotFoundError
+
+    def __create_directory_tree(self, full_path: str, mode: int = 0o744, retry_on_error: bool = True) -> None:
+        """Create directory or overwrite existing one.
+
+        This method will handle a directory tree creation. If there is a collision
+        in directory structure - old directory tree will be removed
+        and creation will be attempted one more time. If the creation fails for the second time
+        the exception will be raised.
+
+        :param full_path: Full directory tree path (eg. one/two/tree) as string.
+        :param mode: Permission mask for the directories.
+        :param retry_on_error: Flag that indicates if there should be a attempt to retry the operation.
+        """
+        try:
+            os.makedirs(full_path, mode=mode)
+        except FileExistsError:
+            # In this case we know that cba of same name and version need to be overwritten
+            if retry_on_error:
+                self.__remove_directory_tree(full_path)
+                self.__create_directory_tree(full_path, mode=mode, retry_on_error=False)
+            else:
+                # This way we won't try for ever if something goes wrong
+                raise ArtifactOverwriteError
+
+    def upload_blueprint(self, cba_bytes: bytes, name: str, version: str) -> None:
+        """Store blueprint file in the repository.
+
+        :param cba_bytes: Bytes to save
+        :param name: Blueprint name
+        :param version: Blueprint version
+        """
+        temporary_file: BytesIO = BytesIO(cba_bytes)
+
+        if not is_zipfile(temporary_file):
+            raise InvalidRequestError
+
+        target_path: str = str(Path(self.base_path.absolute(), name, version))
+        self.__create_directory_tree(target_path)
+
+        with ZipFile(temporary_file, "r") as zip_file:  # type: ZipFile
+            zip_file.extractall(target_path)
+
+    def download_blueprint(self, name: str, version: str) -> bytes:
+        """Download blueprint file from repository.
+
+        This method does the in-memory zipping the files and returns bytes
+
+        :param name: Blueprint name
+        :param version: Blueprint version
+        :return: Zipped Blueprint file bytes
+        """
+        temporary_file: BytesIO = BytesIO()
+        files_path: str = str(Path(self.base_path.absolute(), name, version))
+        if not os.path.exists(files_path):
+            raise ArtifactNotFoundError
+
+        with ZipFile(temporary_file, "w") as zip_file:  # type: ZipFile
+            for directory_name, subdirectory_names, filenames in os.walk(files_path):  # type: str, list, list
+                for filename in filenames:  # type: str
+                    zip_file.write(Path(directory_name, filename))
+
+        # Rewind the fake file to allow reading
+        temporary_file.seek(0)
+
+        zip_as_bytes: bytes = temporary_file.read()
+        temporary_file.close()
+        return zip_as_bytes
+
+    def remove_blueprint(self, name: str, version: str) -> None:
+        """Remove blueprint file from repository.
+
+        :param name: Blueprint name
+        :param version: Blueprint version
+        :raises: FileNotFoundError
+        """
+        files_path: str = str(Path(self.base_path.absolute(), name, version))
+        self.__remove_directory_tree(files_path)
+
+
+class RepositoryStrategy(ABC):
+    """Strategy class.
+
+    It has only one public method `get_repository`, which returns valid repository
+    instance for the the configuration value.
+    You can create many Repository subclasses, but repository clients doesn't have
+    to know which one you use.
+    """
+
+    @classmethod
+    def get_reporitory(cls) -> Repository:
+        """Get the valid repository instance for the configuration value.
+
+        Currently it returns FileRepository because it is an only Repository implementation.
+        """
+        return FileRepository(Path(config["artifactManagerServer"]["fileRepositoryBasePath"]))