diff options
12 files changed, 466 insertions, 44 deletions
diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2421e386c..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "files.exclude": { - "**/.classpath": true, - "**/.project": true, - "**/.settings": true, - "**/.factorypath": true - } -}
\ No newline at end of file diff --git a/ms/py-executor/resource_resolution/README b/ms/py-executor/resource_resolution/README.md index 353600445..3920338ae 100644 --- a/ms/py-executor/resource_resolution/README +++ b/ms/py-executor/resource_resolution/README.md @@ -1,4 +1,4 @@ -# Resource resolution client +# Resource resolution GRPC client ## How to use examples @@ -7,7 +7,7 @@ ``` from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient +from resource_resolution.grpc.client import Client as ResourceResolutionClient def generate_messages(): @@ -45,7 +45,7 @@ if __name__ == "__main__": ``` from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient +from resource_resolution.grpc.client import Client as ResourceResolutionClient def generate_messages(): @@ -84,7 +84,7 @@ if __name__ == "__main__": ``` from proto.BluePrintCommon_pb2 import ActionIdentifiers, CommonHeader from proto.BluePrintProcessing_pb2 import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient +from resource_resolution.grpc.client import Client as ResourceResolutionClient def generate_messages(): @@ -121,7 +121,7 @@ if __name__ == "__main__": ## How to use examples -### Insecure channel +### GRPC insecure channel ``` from resource_resolution.resource_resolution import ResourceResolution, WorkflowExecution, WorkflowExecutionResult @@ -140,4 +140,41 @@ if __name__ == "__main__": print(response.error_message) else: print(response.payload) +``` + +### HTTP retrieve/store template + +``` +from resource_resolution.resource_resolution import ResourceResolution + +if __name__ == "__main__": + # If you want to use only HTTP you don't have to use context manager + r = ResourceResolution( + http_server_port=8081, + http_auth_user="ccsdkapps", + http_auth_pass="ccsdkapps", + http_use_tls=False + ) + r.store_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + result="test") + template = r.retrieve_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + ) + assert template.result == "test" + template.result = "another value" + template.store() + another_template = r.retrieve_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + ) + assert another_template.result == "another_value" ```
\ No newline at end of file diff --git a/ms/py-executor/resource_resolution/grpc/__init__.py b/ms/py-executor/resource_resolution/grpc/__init__.py new file mode 100644 index 000000000..1894680a6 --- /dev/null +++ b/ms/py-executor/resource_resolution/grpc/__init__.py @@ -0,0 +1,16 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .client import Client diff --git a/ms/py-executor/resource_resolution/authorization.py b/ms/py-executor/resource_resolution/grpc/authorization.py index ae5954ecc..ae5954ecc 100644 --- a/ms/py-executor/resource_resolution/authorization.py +++ b/ms/py-executor/resource_resolution/grpc/authorization.py diff --git a/ms/py-executor/resource_resolution/client.py b/ms/py-executor/resource_resolution/grpc/client.py index fee168628..fee168628 100644 --- a/ms/py-executor/resource_resolution/client.py +++ b/ms/py-executor/resource_resolution/grpc/client.py diff --git a/ms/py-executor/resource_resolution/http/__init__.py b/ms/py-executor/resource_resolution/http/__init__.py new file mode 100644 index 000000000..1894680a6 --- /dev/null +++ b/ms/py-executor/resource_resolution/http/__init__.py @@ -0,0 +1,16 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .client import Client diff --git a/ms/py-executor/resource_resolution/http/client.py b/ms/py-executor/resource_resolution/http/client.py new file mode 100644 index 000000000..8bb1e1be4 --- /dev/null +++ b/ms/py-executor/resource_resolution/http/client.py @@ -0,0 +1,96 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import Optional, Tuple + +from requests import Session, request, Request, Response, PreparedRequest + + +class Client: + """HTTP client class.""" + + API_VERSION = "v1" + + def __init__( + self, server_address: str, server_port: int, auth_user: str = None, auth_pass: str = None, use_ssl: bool = False + ) -> None: + """HTTP client class initialization. + + Args: + server_address (str): HTTP server address + server_port (int): HTTP server port + auth_user (str, optional): Username used for authorization. Defaults to None. + auth_pass (str, optional): Password used for authorization. Defaults to None. + use_ssl (bool, optional): Determines if secure connection has to be used. Defaults to False. + """ + self.server_address: str = server_address + self.server_port: int = server_port + self.use_ssl: bool = use_ssl + + self.auth_user: str = auth_user + self.auth_pass: str = auth_pass + + @property + def auth(self) -> Optional[Tuple[str, str]]: + """Authorization data tuple or None. + + Returns None if not both auth_user and auth_pass values are set. + + Returns: + Optional[Tuple[str, str]]: Authorization tuple (auth_user, auth_pass) or None + """ + if all([self.auth_user, self.auth_pass]): + return (self.auth_user, self.auth_pass) + return None + + @property + def protocol(self) -> str: + """Protocol which is going to be used for request call. + + Returns: + str: http or https + """ + if self.use_ssl: + return "https" + return "http" + + @property + def url(self) -> str: + """Url to call requests. + + Returns: + str: Url string + """ + return f"{self.protocol}://{self.server_address}:{self.server_port}/api/{self.API_VERSION}" + + def send_request(self, method: str, endpoint: str, **kwargs) -> Response: + """Send request to server. + + Send request with `method` method to server. Pass any additional values as **kwargs. + + Args: + method (str): HTTP method + endpoint (str): Endpoint to call a request + + Raises: + requests.HTTPError: An HTTP error occurred. + + Returns: + Response: `requests.Response` object. + """ + response: Response = request( + method=method, url=f"{self.url}/{endpoint}", verify=False, auth=self.auth, **kwargs + ) + response.raise_for_status() + return response diff --git a/ms/py-executor/resource_resolution/resource_resolution.py b/ms/py-executor/resource_resolution/resource_resolution.py index e4f162f8f..3b8c19b74 100644 --- a/ms/py-executor/resource_resolution/resource_resolution.py +++ b/ms/py-executor/resource_resolution/resource_resolution.py @@ -13,8 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json +from dataclasses import dataclass, field from enum import Enum, unique from logging import Logger, getLogger +from os import getenv from types import TracebackType from typing import Any, Dict, Generator, Optional, Type @@ -22,7 +25,8 @@ from google.protobuf import json_format from proto.BluePrintProcessing_pb2 import ExecutionServiceInput, ExecutionServiceOutput -from .client import Client +from .grpc import Client as GrpcClient +from .http import Client as HttpClient @unique @@ -179,6 +183,39 @@ class WorkflowExecutionResult: return json_format.MessageToDict(self.execution_output.payload) +@dataclass +class Template: + """Template dataclass. + + Store resolved template data. + It keeps also ResourceResolution object to call `store_template` method. + """ + + resource_resolution: "ResourceResolution" = field(repr=False) + blueprint_name: str + blueprint_version: str + artifact_name: str = None + result: str = None + resolution_key: str = None + resource_type: str = None + resource_id: str = None + + def store(self) -> None: + """Store template using blueprintprocessor HTTP API. + + It uses ResourceResolution `store_template` method. + """ + self.resource_resolution.store_template( + blueprint_name=self.blueprint_name, + blueprint_version=self.blueprint_version, + artifact_name=self.artifact_name, + result=self.result, + resolution_key=self.resolution_key, + resource_type=self.resource_type, + resource_id=self.resource_id, + ) + + class ResourceResolution: """Resource resolution class. @@ -191,20 +228,26 @@ class ResourceResolution: self, *, server_address: str = "127.0.0.1", - server_port: int = "9111", + # GRPC client configuration + grpc_server_port: int = 9111, use_ssl: bool = False, root_certificates: bytes = None, private_key: bytes = None, certificate_chain: bytes = None, - # Authentication header configuration + # Authentication header configuration for GRPC client use_header_auth: bool = False, header_auth_token: str = None, + # HTTP client configuration + http_server_port: int = 8080, + http_auth_user: str = None, + http_auth_pass: str = None, + http_use_ssl: bool = True, ) -> None: """Resource resolution object initialization. Args: server_address (str, optional): gRPC server address. Defaults to "127.0.0.1". - server_port (int, optional): gRPC server address port. Defaults to "9111". + grpc_server_port (int, optional): gRPC server address port. Defaults to 9111. use_ssl (bool, optional): Boolean flag to determine if secure channel should be created or not. Defaults to False. root_certificates (bytes, optional): The PEM-encoded root certificates. None if it shouldn't be used. @@ -216,33 +259,49 @@ class ResourceResolution: use_header_auth (bool, optional): Boolean flag to determine if authorization headed shoud be added for every call or not. Defaults to False. header_auth_token (str, optional): Authorization token value. Defaults to None. + If no value is provided "AUTH_TOKEN" environment variable will be used. + http_server_port (int, optional): HTTP server address port. Defaults to 8080. + http_auth_user (str, optional): Username used for HTTP requests authorization. Defaults to None. + If no value is provided "API_USERNAME" environment variable will be used. + http_auth_pass (str, optional): Password used for HTTP requests authorization. Defaults to None. + If no value is provided "API_PASSWORD" environment variable will be used. + http_use_ssl (bool, optional): Determines if secure connection should be used for HTTP requests. + Defaults to False. """ # Logger self.logger: Logger = getLogger(__name__) - # Client settings - self.client_server_address: str = server_address - self.client_server_port: str = server_port - self.client_use_ssl: bool = use_ssl - self.client_root_certificates: bytes = root_certificates - self.client_private_key: bytes = private_key - self.client_certificate_chain: bytes = certificate_chain - self.client_use_header_auth: bool = use_header_auth - self.client_header_auth_token: str = header_auth_token - self.client: Client = None + # GrpcClient settings + self.grpc_client_server_address: str = server_address + self.grpc_client_server_port: str = grpc_server_port + self.grpc_client_use_ssl: bool = use_ssl + self.grpc_client_root_certificates: bytes = root_certificates + self.grpc_client_private_key: bytes = private_key + self.grpc_client_certificate_chain: bytes = certificate_chain + self.grpc_client_use_header_auth: bool = use_header_auth + self.grpc_client_header_auth_token: str = header_auth_token or getenv("AUTH_TOKEN") + self.grpc_client: GrpcClient = None + # HttpClient settings + self.http_client: HttpClient = HttpClient( + server_address, + server_port=http_server_port, + auth_user=http_auth_user or getenv("API_USERNAME"), + auth_pass=http_auth_pass or getenv("API_PASSWORD"), + use_ssl=http_use_ssl, + ) def __enter__(self) -> "ResourceResolution": """Enter ResourceResolution instance context. - Client connection is created. + GrpcClient connection is created. """ - self.client = Client( - server_address=f"{self.client_server_address}:{self.client_server_port}", - use_ssl=self.client_use_ssl, - root_certificates=self.client_root_certificates, - private_key=self.client_private_key, - certificate_chain=self.client_certificate_chain, - use_header_auth=self.client_use_header_auth, - header_auth_token=self.client_header_auth_token, + self.grpc_client = GrpcClient( + server_address=f"{self.grpc_client_server_address}:{self.grpc_client_server_port}", + use_ssl=self.grpc_client_use_ssl, + root_certificates=self.grpc_client_root_certificates, + private_key=self.grpc_client_private_key, + certificate_chain=self.grpc_client_certificate_chain, + use_header_auth=self.grpc_client_use_header_auth, + header_auth_token=self.grpc_client_header_auth_token, ) return self @@ -254,9 +313,9 @@ class ResourceResolution: ) -> None: """Exit ResourceResolution instance context. - Client connection is closed. + GrpcClient connection is closed. """ - self.client.close() + self.grpc_client.close() def execute_workflows(self, *workflows: WorkflowExecution) -> Generator[WorkflowExecutionResult, None, None]: """Execute provided workflows. @@ -270,7 +329,7 @@ class ResourceResolution: AttributeError: Raises if client object is not created. It occurs only if you not uses context manager. Then user have to create client instance for ResourceResolution object by himself calling: ``` - resource_resoulution.client = Client( + resource_resoulution.client = GrpcClient( server_address=f"{resource_resoulution.client_server_address}:{resource_resoulution.client_server_port}", use_ssl=resource_resoulution.client_use_ssl, root_certificates=resource_resoulution.client_root_certificates, @@ -287,8 +346,112 @@ class ResourceResolution: with both WorkflowExection object and server response for it's request. """ self.logger.debug("Execute workflows") - if not self.client: + if not self.grpc_client: raise AttributeError("gRPC client not connected") - for response, workflow in zip(self.client.process((workflow.message for workflow in workflows)), workflows): + for response, workflow in zip( + self.grpc_client.process((workflow.message for workflow in workflows)), workflows + ): yield WorkflowExecutionResult(workflow, response) + + def _check_template_resolve_params( + self, resolution_key: str = None, resource_type: str = None, resource_id: str = None + ): + """Check template API request parameters. + + It's possible to store/retrieve templates using pair of artifact name and resolution key OR + resource type and resource id. This method checks if valid combination of parameters were used. + + Args: + resolution_key (str, optional): resolutionKey HTTP request parameter value. Defaults to None. + resource_type (str, optional): resourceType HTTP request parameter value. Defaults to None. + resource_id (str, optional): resourceId HTTP request parameter value. Defaults to None. + + Raises: + AttributeError: Invalid combination of parametes used + """ + if not any([resolution_key, all([resource_type, resource_id])]): + raise AttributeError( + "To store/retrieve template resolution_key and artifact_name or both resource_type and resource_id have to be provided" + ) + + def store_template( + self, + blueprint_name: str, + blueprint_version: str, + result: str, + artifact_name: str, + resolution_key: str = None, + resource_type: str = None, + resource_id: str = None, + ) -> None: + """Store template using blueprintprocessor HTTP API. + + Prepare and send a request to store resolved template using blueprint name, blueprint version + and pair of artifact name and resolution key OR resource type and resource id. + + Method returns Template dataclass, which stores all template data and can be used to update + it's result. + + Args: + blueprint_name (str): Blueprint name + blueprint_version (str): Blueprint version + result (str): Template result + artifact_name (str): Artifact name + resolution_key (str, optional): Resolution key. Defaults to None. + resource_type (str, optional): Resource type. Defaults to None. + resource_id (str, optional): Resource ID. Defaults to None. + """ + self.logger.debug("Store template") + self._check_template_resolve_params(resolution_key, resource_type, resource_id) + base_endpoint: str = f"template/{blueprint_name}/{blueprint_version}" + if resolution_key and artifact_name: + endpoint: str = f"{base_endpoint}/{artifact_name}/{resolution_key}" + else: + endpoint: str = f"{base_endpoint}/{resource_type}/{resource_id}" + response = self.http_client.send_request( + "POST", endpoint, headers={"Content-Type": "application/json"}, data=json.dumps({"result": result}) + ) + + def retrieve_template( + self, + blueprint_name: str, + blueprint_version: str, + artifact_name: str, + resolution_key: str = None, + resource_type: str = None, + resource_id: str = None, + ) -> Template: + """Get stored template using blueprintprocessor's HTTP API. + + Prepare and send a request to retrieve resolved template using blueprint name, blueprint version + and pair of artifact name and resolution key OR resource type and resource id. + + Args: + blueprint_name (str): Blueprint name + blueprint_version (str): Blueprint version + artifact_name (str): Artifact name + resolution_key (str, optional): Resolution key. Defaults to None. + resource_type (str, optional): Resource type. Defaults to None. + resource_id (str, optional): Resource ID. Defaults to None. + """ + self.logger.debug("Retrieve template") + self._check_template_resolve_params(resolution_key, resource_type, resource_id) + params: dict = {"bpName": blueprint_name, "bpVersion": blueprint_version, "artifactName": artifact_name} + if resolution_key: + params.update({"resolutionKey": resolution_key}) + else: + params.update({"resourceType": resource_type, "resourceId": resource_id}) + response = self.http_client.send_request( + "GET", "template", headers={"Accept": "application/json"}, params=params + ) + return Template( + resource_resolution=self, + blueprint_name=blueprint_name, + blueprint_version=blueprint_version, + artifact_name=artifact_name, + resolution_key=resolution_key, + resource_type=resource_type, + resource_id=resource_id, + result=response.json()["result"], + ) diff --git a/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py b/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py index 4b03f0b36..734059f3d 100644 --- a/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py +++ b/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py @@ -17,7 +17,7 @@ from unittest.mock import MagicMock, _Call import pytest -from resource_resolution.authorization import AuthTokenInterceptor, NewClientCallDetails +from resource_resolution.grpc.authorization import AuthTokenInterceptor, NewClientCallDetails def test_resource_resolution_auth_token_interceptor(): diff --git a/ms/py-executor/resource_resolution/tests/client_test.py b/ms/py-executor/resource_resolution/tests/grpc_client_test.py index 2b94220f6..8217b0f25 100644 --- a/ms/py-executor/resource_resolution/tests/client_test.py +++ b/ms/py-executor/resource_resolution/tests/grpc_client_test.py @@ -15,10 +15,10 @@ limitations under the License. from unittest.mock import MagicMock, patch -from resource_resolution.client import Client +from resource_resolution.grpc.client import Client -@patch("resource_resolution.client.insecure_channel") +@patch("resource_resolution.grpc.client.insecure_channel") def test_resource_resoulution_insecure_channel(insecure_channel_mock: MagicMock): """Test if insecure_channel connection is called.""" with patch.object(Client, "close") as client_close_method_mock: # Type MagicMock diff --git a/ms/py-executor/resource_resolution/tests/http_client_test.py b/ms/py-executor/resource_resolution/tests/http_client_test.py new file mode 100644 index 000000000..2279fde7a --- /dev/null +++ b/ms/py-executor/resource_resolution/tests/http_client_test.py @@ -0,0 +1,38 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from unittest.mock import MagicMock, patch + +from resource_resolution.http.client import Client + + +@patch("resource_resolution.http.client.request") +def test_http_client(request_mock): + c = Client("127.0.0.1", 8080) + assert c.auth is None + c = Client("127.0.0.1", 8080, auth_user="user") + assert c.auth is None + c = Client("127.0.0.1", 8080, auth_user="user", auth_pass="pass") + assert c.auth == ("user", "pass") + + assert c.protocol == "http" + assert c.url == "http://127.0.0.1:8080/api/v1" + + c = Client("127.0.0.1", 8081, use_ssl=True) + assert c.protocol == "https" + assert c.url == "https://127.0.0.1:8081/api/v1" + + c.send_request("GET", "something") + request_mock.assert_called_once_with(method="GET", url=f"{c.url}/something", verify=False, auth=None) diff --git a/ms/py-executor/resource_resolution/tests/resource_resolution_test.py b/ms/py-executor/resource_resolution/tests/resource_resolution_test.py index 8a41357e6..274802279 100644 --- a/ms/py-executor/resource_resolution/tests/resource_resolution_test.py +++ b/ms/py-executor/resource_resolution/tests/resource_resolution_test.py @@ -13,12 +13,17 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json +from unittest.mock import patch, MagicMock + from google.protobuf import json_format from pytest import raises from resource_resolution.resource_resolution import ( ExecutionServiceInput, ExecutionServiceOutput, + ResourceResolution, + Template, WorkflowExecution, WorkflowExecutionResult, WorkflowMode, @@ -103,3 +108,62 @@ def test_workflow_execution_result_class(): execution_output.status.code = 500 assert execution_result.has_error assert execution_result.error_message == "" + + +def test_resource_resolution_check_resolve_params(): + """Check values of potentially HTTP parameters.""" + rr = ResourceResolution() + with raises(AttributeError): + rr._check_template_resolve_params() + rr._check_template_resolve_params(resource_type="test") + rr._check_template_resolve_params(resource_id="test") + rr._check_template_resolve_params(resolution_key="test") + rr._check_template_resolve_params(resource_type="test", resource_id="test") + + +def test_store_template(): + """Test store_template method. + + Checks if http_client send_request method is called with valid parameters. + """ + rr = ResourceResolution(server_address="127.0.0.1", http_server_port=8080) + rr.http_client = MagicMock() + rr.store_template( + blueprint_name="test_blueprint_name", + blueprint_version="test_blueprint_version", + artifact_name="test_artifact_name", + resolution_key="test_resolution_key", + result="test_result", + ) + rr.http_client.send_request.assert_called_once_with( + "POST", + "template/test_blueprint_name/test_blueprint_version/test_artifact_name/test_resolution_key", + data=json.dumps({"result": "test_result"}), + headers={"Content-Type": "application/json"}, + ) + + +def test_retrieve_template(): + """Test retrieve_template method. + + Checks if http_client send_request method is called with valid parameters. + """ + rr = ResourceResolution(server_address="127.0.0.1", http_server_port=8080) + rr.http_client = MagicMock() + rr.retrieve_template( + blueprint_name="test_blueprint_name", + blueprint_version="test_blueprint_version", + artifact_name="test_artifact_name", + resolution_key="test_resolution_key", + ) + rr.http_client.send_request.assert_called_once_with( + "GET", + "template", + params={ + "bpName": "test_blueprint_name", + "bpVersion": "test_blueprint_version", + "artifactName": "test_artifact_name", + "resolutionKey": "test_resolution_key", + }, + headers={"Accept": "application/json"}, + ) |