From daf7bf3b0726c9574f9f1b7aa34af4f199ee32c3 Mon Sep 17 00:00:00 2001 From: Michal Jagiello Date: Fri, 27 Mar 2020 12:16:22 +0000 Subject: PyExecutor ResourceResolution store/retrieve templates Issue-ID: CCSDK-2156 Signed-off-by: Michal Jagiello Change-Id: I59df2772d004e349532a1b42c4e4abd367c13256 --- .vscode/settings.json | 8 - ms/py-executor/resource_resolution/README | 143 -------------- ms/py-executor/resource_resolution/README.md | 180 +++++++++++++++++ .../resource_resolution/authorization.py | 64 ------ ms/py-executor/resource_resolution/client.py | 112 ----------- .../resource_resolution/grpc/__init__.py | 16 ++ .../resource_resolution/grpc/authorization.py | 64 ++++++ ms/py-executor/resource_resolution/grpc/client.py | 112 +++++++++++ .../resource_resolution/http/__init__.py | 16 ++ ms/py-executor/resource_resolution/http/client.py | 96 +++++++++ .../resource_resolution/resource_resolution.py | 219 ++++++++++++++++++--- .../tests/authorization_interceptor_test.py | 2 +- .../resource_resolution/tests/client_test.py | 28 --- .../resource_resolution/tests/grpc_client_test.py | 28 +++ .../resource_resolution/tests/http_client_test.py | 38 ++++ .../tests/resource_resolution_test.py | 64 ++++++ 16 files changed, 806 insertions(+), 384 deletions(-) delete mode 100644 .vscode/settings.json delete mode 100644 ms/py-executor/resource_resolution/README create mode 100644 ms/py-executor/resource_resolution/README.md delete mode 100644 ms/py-executor/resource_resolution/authorization.py delete mode 100644 ms/py-executor/resource_resolution/client.py create mode 100644 ms/py-executor/resource_resolution/grpc/__init__.py create mode 100644 ms/py-executor/resource_resolution/grpc/authorization.py create mode 100644 ms/py-executor/resource_resolution/grpc/client.py create mode 100644 ms/py-executor/resource_resolution/http/__init__.py create mode 100644 ms/py-executor/resource_resolution/http/client.py delete mode 100644 ms/py-executor/resource_resolution/tests/client_test.py create mode 100644 ms/py-executor/resource_resolution/tests/grpc_client_test.py create mode 100644 ms/py-executor/resource_resolution/tests/http_client_test.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2421e386c..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "files.exclude": { - "**/.classpath": true, - "**/.project": true, - "**/.settings": true, - "**/.factorypath": true - } -} \ No newline at end of file diff --git a/ms/py-executor/resource_resolution/README b/ms/py-executor/resource_resolution/README deleted file mode 100644 index 353600445..000000000 --- a/ms/py-executor/resource_resolution/README +++ /dev/null @@ -1,143 +0,0 @@ -# Resource resolution client - -## How to use examples - -### Insecure channel - -``` -from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader -from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient - - -def generate_messages(): - commonHeader = CommonHeader() - commonHeader.requestId = "1234" - commonHeader.subRequestId = "1234-1" - commonHeader.originatorId = "CDS" - - actionIdentifiers = ActionIdentifiers() - actionIdentifiers.blueprintName = "sample-cba" - actionIdentifiers.blueprintVersion = "1.0.0" - actionIdentifiers.actionName = "SampleScript" - - input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) - - commonHeader2 = CommonHeader() - commonHeader2.requestId = "1235" - commonHeader2.subRequestId = "1234-2" - commonHeader2.originatorId = "CDS" - - input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) - - yield from [input, input2] - - -if __name__ == "__main__": - with ResourceResolutionClient("localhost:50052") as client: - for response in client.process(generate_messages()): - print(response) - -``` - -### Secure channel - -``` -from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader -from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient - - -def generate_messages(): - commonHeader = CommonHeader() - commonHeader.requestId = "1234" - commonHeader.subRequestId = "1234-1" - commonHeader.originatorId = "CDS" - - actionIdentifiers = ActionIdentifiers() - actionIdentifiers.blueprintName = "sample-cba" - actionIdentifiers.blueprintVersion = "1.0.0" - actionIdentifiers.actionName = "SampleScript" - - input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) - - commonHeader2 = CommonHeader() - commonHeader2.requestId = "1235" - commonHeader2.subRequestId = "1234-2" - commonHeader2.originatorId = "CDS" - - input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) - - yield from [input, input2] - - -if __name__ == "__main__": - with open("certs/py-executor/py-executor-chain.pem", "rb") as f: - with ResourceResolutionClient("localhost:50052", use_ssl=True, root_certificates=f.read()) as client: - for response in client.process(generate_messages()): - print(response) - -``` - -### Authorizarion header - -``` -from proto.BluePrintCommon_pb2 import ActionIdentifiers, CommonHeader -from proto.BluePrintProcessing_pb2 import ExecutionServiceInput -from resource_resolution.client import Client as ResourceResolutionClient - - -def generate_messages(): - commonHeader = CommonHeader() - commonHeader.requestId = "1234" - commonHeader.subRequestId = "1234-1" - commonHeader.originatorId = "CDS" - - actionIdentifiers = ActionIdentifiers() - actionIdentifiers.blueprintName = "sample-cba" - actionIdentifiers.blueprintVersion = "1.0.0" - actionIdentifiers.actionName = "SampleScript" - - input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) - - commonHeader2 = CommonHeader() - commonHeader2.requestId = "1235" - commonHeader2.subRequestId = "1234-2" - commonHeader2.originatorId = "CDS" - - input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) - - yield from [input, input2] - - -if __name__ == "__main__": - with ResourceResolutionClient("127.0.0.1:9111", use_header_auth=True, header_auth_token="Token test") as client: - for response in client.process(generate_messages()): - print(response) - -``` - -# ResourceResoulution helper class - -## How to use examples - -### Insecure channel - -``` -from resource_resolution.resource_resolution import ResourceResolution, WorkflowExecution, WorkflowExecutionResult - - -if __name__ == "__main__": - with ResourceResolution(use_header_auth=True, header_auth_token="Basic token") as rr: - for response in rr.execute_workflows( # type: WorkflowExecutionResult - WorkflowExecution( - blueprint_name="blueprintName", - blueprint_version="1.0", - workflow_name="resource-assignment" - ) - ): - if response.has_error: - print(response.error_message) - else: - print(response.payload) -``` \ No newline at end of file diff --git a/ms/py-executor/resource_resolution/README.md b/ms/py-executor/resource_resolution/README.md new file mode 100644 index 000000000..3920338ae --- /dev/null +++ b/ms/py-executor/resource_resolution/README.md @@ -0,0 +1,180 @@ +# Resource resolution GRPC client + +## How to use examples + +### Insecure channel + +``` +from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader +from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput +from resource_resolution.grpc.client import Client as ResourceResolutionClient + + +def generate_messages(): + commonHeader = CommonHeader() + commonHeader.requestId = "1234" + commonHeader.subRequestId = "1234-1" + commonHeader.originatorId = "CDS" + + actionIdentifiers = ActionIdentifiers() + actionIdentifiers.blueprintName = "sample-cba" + actionIdentifiers.blueprintVersion = "1.0.0" + actionIdentifiers.actionName = "SampleScript" + + input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) + + commonHeader2 = CommonHeader() + commonHeader2.requestId = "1235" + commonHeader2.subRequestId = "1234-2" + commonHeader2.originatorId = "CDS" + + input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) + + yield from [input, input2] + + +if __name__ == "__main__": + with ResourceResolutionClient("localhost:50052") as client: + for response in client.process(generate_messages()): + print(response) + +``` + +### Secure channel + +``` +from proto.BluePrintCommon_pb2_grpc import ActionIdentifiers, CommonHeader +from proto.BluePrintProcessing_pb2_grpc import ExecutionServiceInput +from resource_resolution.grpc.client import Client as ResourceResolutionClient + + +def generate_messages(): + commonHeader = CommonHeader() + commonHeader.requestId = "1234" + commonHeader.subRequestId = "1234-1" + commonHeader.originatorId = "CDS" + + actionIdentifiers = ActionIdentifiers() + actionIdentifiers.blueprintName = "sample-cba" + actionIdentifiers.blueprintVersion = "1.0.0" + actionIdentifiers.actionName = "SampleScript" + + input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) + + commonHeader2 = CommonHeader() + commonHeader2.requestId = "1235" + commonHeader2.subRequestId = "1234-2" + commonHeader2.originatorId = "CDS" + + input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) + + yield from [input, input2] + + +if __name__ == "__main__": + with open("certs/py-executor/py-executor-chain.pem", "rb") as f: + with ResourceResolutionClient("localhost:50052", use_ssl=True, root_certificates=f.read()) as client: + for response in client.process(generate_messages()): + print(response) + +``` + +### Authorizarion header + +``` +from proto.BluePrintCommon_pb2 import ActionIdentifiers, CommonHeader +from proto.BluePrintProcessing_pb2 import ExecutionServiceInput +from resource_resolution.grpc.client import Client as ResourceResolutionClient + + +def generate_messages(): + commonHeader = CommonHeader() + commonHeader.requestId = "1234" + commonHeader.subRequestId = "1234-1" + commonHeader.originatorId = "CDS" + + actionIdentifiers = ActionIdentifiers() + actionIdentifiers.blueprintName = "sample-cba" + actionIdentifiers.blueprintVersion = "1.0.0" + actionIdentifiers.actionName = "SampleScript" + + input = ExecutionServiceInput(commonHeader=commonHeader, actionIdentifiers=actionIdentifiers) + + commonHeader2 = CommonHeader() + commonHeader2.requestId = "1235" + commonHeader2.subRequestId = "1234-2" + commonHeader2.originatorId = "CDS" + + input2 = ExecutionServiceInput(commonHeader=commonHeader2, actionIdentifiers=actionIdentifiers) + + yield from [input, input2] + + +if __name__ == "__main__": + with ResourceResolutionClient("127.0.0.1:9111", use_header_auth=True, header_auth_token="Token test") as client: + for response in client.process(generate_messages()): + print(response) + +``` + +# ResourceResoulution helper class + +## How to use examples + +### GRPC insecure channel + +``` +from resource_resolution.resource_resolution import ResourceResolution, WorkflowExecution, WorkflowExecutionResult + + +if __name__ == "__main__": + with ResourceResolution(use_header_auth=True, header_auth_token="Basic token") as rr: + for response in rr.execute_workflows( # type: WorkflowExecutionResult + WorkflowExecution( + blueprint_name="blueprintName", + blueprint_version="1.0", + workflow_name="resource-assignment" + ) + ): + if response.has_error: + print(response.error_message) + else: + print(response.payload) +``` + +### HTTP retrieve/store template + +``` +from resource_resolution.resource_resolution import ResourceResolution + +if __name__ == "__main__": + # If you want to use only HTTP you don't have to use context manager + r = ResourceResolution( + http_server_port=8081, + http_auth_user="ccsdkapps", + http_auth_pass="ccsdkapps", + http_use_tls=False + ) + r.store_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + result="test") + template = r.retrieve_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + ) + assert template.result == "test" + template.result = "another value" + template.store() + another_template = r.retrieve_template( + blueprint_name="blueprintName", + blueprint_version="1.0.0", + artifact_name="test", + resolution_key="test", + ) + assert another_template.result == "another_value" +``` \ No newline at end of file diff --git a/ms/py-executor/resource_resolution/authorization.py b/ms/py-executor/resource_resolution/authorization.py deleted file mode 100644 index ae5954ecc..000000000 --- a/ms/py-executor/resource_resolution/authorization.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Copyright 2020 Deutsche Telekom. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from collections import namedtuple -from typing import Any, Callable, List - -from grpc import ClientCallDetails, StreamStreamClientInterceptor - - -class NewClientCallDetails( - namedtuple("_ClientCallDetails", ("method", "timeout", "metadata", "credentials")), ClientCallDetails -): - """Namedtuple class to store metadata. - - It's impossible to change original metadata in ClientCallDetails object - passed as a parameter to intercept method, so this class is going to get - original metadata tuple and add the authorization one. - """ - - pass - - -class AuthTokenInterceptor(StreamStreamClientInterceptor): - """Interceptor class to set authorization header. - - Set authorization header (but it can be any header also) for a gRPC call. - """ - - def __init__(self, token: str, header: str = "authorization") -> None: - """Initialize interceptor. - - Set token and header which should be set into call. By default header is "authorization". - Header have to be lowercase. - - Args: - token (str): Token value to be set. - header (str, optional): Header name. It must be lowercase. Defaults to "authorization". - """ - self.token: str = token - if not header.islower(): - raise ValueError("Header must be lowercase.") - self.header: str = header - - def intercept_stream_stream( - self, continuation: Callable, client_call_details: ClientCallDetails, request_iterator: Any - ) -> Any: - """Add header into metadata.""" - metadata: List = list(client_call_details.metadata) if client_call_details.metadata is not None else [] - metadata.append((self.header, self.token,)) - new_client_call_details: NewClientCallDetails = NewClientCallDetails( - client_call_details.method, client_call_details.timeout, metadata, client_call_details.credentials - ) - return continuation(new_client_call_details, request_iterator) diff --git a/ms/py-executor/resource_resolution/client.py b/ms/py-executor/resource_resolution/client.py deleted file mode 100644 index fee168628..000000000 --- a/ms/py-executor/resource_resolution/client.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Copyright 2019 Deutsche Telekom. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from logging import Logger, getLogger -from types import TracebackType -from typing import Iterable, Optional, Type - -from grpc import ( - Channel, - insecure_channel, - intercept_channel, - secure_channel, - ssl_channel_credentials, -) -from proto.BluePrintProcessing_pb2 import ExecutionServiceInput, ExecutionServiceOutput -from proto.BluePrintProcessing_pb2_grpc import BluePrintProcessingServiceStub - -from .authorization import AuthTokenInterceptor - - -class Client: - """Resource resoulution client class.""" - - def __init__( - self, - server_address: str, - *, - # TLS/SSL configuration - use_ssl: bool = False, - root_certificates: bytes = None, - private_key: bytes = None, - certificate_chain: bytes = None, - # Authentication header configuration - use_header_auth: bool = False, - header_auth_token: str = None, - ) -> None: - """Client class initialization. - - :param server_address: Address to server to connect. - :param use_ssl: Boolean flag to determine if secure channel should be created or not. Keyword argument. - :param root_certificates: The PEM-encoded root certificates. None if it shouldn't be used. Keyword argument. - :param private_key: The PEM-encoded private key as a byte string, or None if no private key should be used. - Keyword argument. - :param certificate_chain: The PEM-encoded certificate chain as a byte string to use or or None if - no certificate chain should be used. Keyword argument. - :param use_header_auth: Boolean flag to determine if authorization headed shoud be added for every call or not. - Keyword argument. - :param header_auth_token: Authorization token value. Keyword argument. - """ - self.logger: Logger = getLogger(__name__) - if use_ssl: - self.channel: Channel = secure_channel( - server_address, ssl_channel_credentials(root_certificates, private_key, certificate_chain) - ) - self.logger.debug(f"Create secure channel to connect with {server_address}") - else: - self.channel: Channel = insecure_channel(server_address) - self.logger.debug(f"Create insecure channel to connect to {server_address}") - if use_header_auth: - self.channel: Channel = intercept_channel(self.channel, AuthTokenInterceptor(header_auth_token)) - self.stub: BluePrintProcessingServiceStub = BluePrintProcessingServiceStub(self.channel) - - def close(self) -> None: - """Close client session. - - Closes client's channel. - """ - self.logger.debug("Close channel connection") - self.channel.close() - - def __enter__(self) -> Channel: - """Enter Client instance context. - - Return Client instance. In the context user can call methods to communicate with server. - On exit connection with the server is going to be closed. - """ - self.logger.debug("Enter Client instance context") - return self - - def __exit__( - self, - unused_exc_type: Optional[Type[BaseException]], - unused_exc_value: Optional[BaseException], - unused_traceback: Optional[TracebackType], - ) -> None: - """Exit Client instance context. - - Close connection with the server. - """ - self.logger.debug("Exit Client instance context") - self.close() - - def process(self, messages: Iterable[ExecutionServiceInput]) -> Iterable[ExecutionServiceOutput]: - """Send messages to server and return responses. - - :param messages: Iterable messages to send - :return: Iterable responses - """ - for message in self.stub.process(messages): - self.logger.debug(f"Get response message: {message}") - yield message diff --git a/ms/py-executor/resource_resolution/grpc/__init__.py b/ms/py-executor/resource_resolution/grpc/__init__.py new file mode 100644 index 000000000..1894680a6 --- /dev/null +++ b/ms/py-executor/resource_resolution/grpc/__init__.py @@ -0,0 +1,16 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .client import Client diff --git a/ms/py-executor/resource_resolution/grpc/authorization.py b/ms/py-executor/resource_resolution/grpc/authorization.py new file mode 100644 index 000000000..ae5954ecc --- /dev/null +++ b/ms/py-executor/resource_resolution/grpc/authorization.py @@ -0,0 +1,64 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from collections import namedtuple +from typing import Any, Callable, List + +from grpc import ClientCallDetails, StreamStreamClientInterceptor + + +class NewClientCallDetails( + namedtuple("_ClientCallDetails", ("method", "timeout", "metadata", "credentials")), ClientCallDetails +): + """Namedtuple class to store metadata. + + It's impossible to change original metadata in ClientCallDetails object + passed as a parameter to intercept method, so this class is going to get + original metadata tuple and add the authorization one. + """ + + pass + + +class AuthTokenInterceptor(StreamStreamClientInterceptor): + """Interceptor class to set authorization header. + + Set authorization header (but it can be any header also) for a gRPC call. + """ + + def __init__(self, token: str, header: str = "authorization") -> None: + """Initialize interceptor. + + Set token and header which should be set into call. By default header is "authorization". + Header have to be lowercase. + + Args: + token (str): Token value to be set. + header (str, optional): Header name. It must be lowercase. Defaults to "authorization". + """ + self.token: str = token + if not header.islower(): + raise ValueError("Header must be lowercase.") + self.header: str = header + + def intercept_stream_stream( + self, continuation: Callable, client_call_details: ClientCallDetails, request_iterator: Any + ) -> Any: + """Add header into metadata.""" + metadata: List = list(client_call_details.metadata) if client_call_details.metadata is not None else [] + metadata.append((self.header, self.token,)) + new_client_call_details: NewClientCallDetails = NewClientCallDetails( + client_call_details.method, client_call_details.timeout, metadata, client_call_details.credentials + ) + return continuation(new_client_call_details, request_iterator) diff --git a/ms/py-executor/resource_resolution/grpc/client.py b/ms/py-executor/resource_resolution/grpc/client.py new file mode 100644 index 000000000..fee168628 --- /dev/null +++ b/ms/py-executor/resource_resolution/grpc/client.py @@ -0,0 +1,112 @@ +"""Copyright 2019 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from logging import Logger, getLogger +from types import TracebackType +from typing import Iterable, Optional, Type + +from grpc import ( + Channel, + insecure_channel, + intercept_channel, + secure_channel, + ssl_channel_credentials, +) +from proto.BluePrintProcessing_pb2 import ExecutionServiceInput, ExecutionServiceOutput +from proto.BluePrintProcessing_pb2_grpc import BluePrintProcessingServiceStub + +from .authorization import AuthTokenInterceptor + + +class Client: + """Resource resoulution client class.""" + + def __init__( + self, + server_address: str, + *, + # TLS/SSL configuration + use_ssl: bool = False, + root_certificates: bytes = None, + private_key: bytes = None, + certificate_chain: bytes = None, + # Authentication header configuration + use_header_auth: bool = False, + header_auth_token: str = None, + ) -> None: + """Client class initialization. + + :param server_address: Address to server to connect. + :param use_ssl: Boolean flag to determine if secure channel should be created or not. Keyword argument. + :param root_certificates: The PEM-encoded root certificates. None if it shouldn't be used. Keyword argument. + :param private_key: The PEM-encoded private key as a byte string, or None if no private key should be used. + Keyword argument. + :param certificate_chain: The PEM-encoded certificate chain as a byte string to use or or None if + no certificate chain should be used. Keyword argument. + :param use_header_auth: Boolean flag to determine if authorization headed shoud be added for every call or not. + Keyword argument. + :param header_auth_token: Authorization token value. Keyword argument. + """ + self.logger: Logger = getLogger(__name__) + if use_ssl: + self.channel: Channel = secure_channel( + server_address, ssl_channel_credentials(root_certificates, private_key, certificate_chain) + ) + self.logger.debug(f"Create secure channel to connect with {server_address}") + else: + self.channel: Channel = insecure_channel(server_address) + self.logger.debug(f"Create insecure channel to connect to {server_address}") + if use_header_auth: + self.channel: Channel = intercept_channel(self.channel, AuthTokenInterceptor(header_auth_token)) + self.stub: BluePrintProcessingServiceStub = BluePrintProcessingServiceStub(self.channel) + + def close(self) -> None: + """Close client session. + + Closes client's channel. + """ + self.logger.debug("Close channel connection") + self.channel.close() + + def __enter__(self) -> Channel: + """Enter Client instance context. + + Return Client instance. In the context user can call methods to communicate with server. + On exit connection with the server is going to be closed. + """ + self.logger.debug("Enter Client instance context") + return self + + def __exit__( + self, + unused_exc_type: Optional[Type[BaseException]], + unused_exc_value: Optional[BaseException], + unused_traceback: Optional[TracebackType], + ) -> None: + """Exit Client instance context. + + Close connection with the server. + """ + self.logger.debug("Exit Client instance context") + self.close() + + def process(self, messages: Iterable[ExecutionServiceInput]) -> Iterable[ExecutionServiceOutput]: + """Send messages to server and return responses. + + :param messages: Iterable messages to send + :return: Iterable responses + """ + for message in self.stub.process(messages): + self.logger.debug(f"Get response message: {message}") + yield message diff --git a/ms/py-executor/resource_resolution/http/__init__.py b/ms/py-executor/resource_resolution/http/__init__.py new file mode 100644 index 000000000..1894680a6 --- /dev/null +++ b/ms/py-executor/resource_resolution/http/__init__.py @@ -0,0 +1,16 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .client import Client diff --git a/ms/py-executor/resource_resolution/http/client.py b/ms/py-executor/resource_resolution/http/client.py new file mode 100644 index 000000000..8bb1e1be4 --- /dev/null +++ b/ms/py-executor/resource_resolution/http/client.py @@ -0,0 +1,96 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import Optional, Tuple + +from requests import Session, request, Request, Response, PreparedRequest + + +class Client: + """HTTP client class.""" + + API_VERSION = "v1" + + def __init__( + self, server_address: str, server_port: int, auth_user: str = None, auth_pass: str = None, use_ssl: bool = False + ) -> None: + """HTTP client class initialization. + + Args: + server_address (str): HTTP server address + server_port (int): HTTP server port + auth_user (str, optional): Username used for authorization. Defaults to None. + auth_pass (str, optional): Password used for authorization. Defaults to None. + use_ssl (bool, optional): Determines if secure connection has to be used. Defaults to False. + """ + self.server_address: str = server_address + self.server_port: int = server_port + self.use_ssl: bool = use_ssl + + self.auth_user: str = auth_user + self.auth_pass: str = auth_pass + + @property + def auth(self) -> Optional[Tuple[str, str]]: + """Authorization data tuple or None. + + Returns None if not both auth_user and auth_pass values are set. + + Returns: + Optional[Tuple[str, str]]: Authorization tuple (auth_user, auth_pass) or None + """ + if all([self.auth_user, self.auth_pass]): + return (self.auth_user, self.auth_pass) + return None + + @property + def protocol(self) -> str: + """Protocol which is going to be used for request call. + + Returns: + str: http or https + """ + if self.use_ssl: + return "https" + return "http" + + @property + def url(self) -> str: + """Url to call requests. + + Returns: + str: Url string + """ + return f"{self.protocol}://{self.server_address}:{self.server_port}/api/{self.API_VERSION}" + + def send_request(self, method: str, endpoint: str, **kwargs) -> Response: + """Send request to server. + + Send request with `method` method to server. Pass any additional values as **kwargs. + + Args: + method (str): HTTP method + endpoint (str): Endpoint to call a request + + Raises: + requests.HTTPError: An HTTP error occurred. + + Returns: + Response: `requests.Response` object. + """ + response: Response = request( + method=method, url=f"{self.url}/{endpoint}", verify=False, auth=self.auth, **kwargs + ) + response.raise_for_status() + return response diff --git a/ms/py-executor/resource_resolution/resource_resolution.py b/ms/py-executor/resource_resolution/resource_resolution.py index e4f162f8f..3b8c19b74 100644 --- a/ms/py-executor/resource_resolution/resource_resolution.py +++ b/ms/py-executor/resource_resolution/resource_resolution.py @@ -13,8 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json +from dataclasses import dataclass, field from enum import Enum, unique from logging import Logger, getLogger +from os import getenv from types import TracebackType from typing import Any, Dict, Generator, Optional, Type @@ -22,7 +25,8 @@ from google.protobuf import json_format from proto.BluePrintProcessing_pb2 import ExecutionServiceInput, ExecutionServiceOutput -from .client import Client +from .grpc import Client as GrpcClient +from .http import Client as HttpClient @unique @@ -179,6 +183,39 @@ class WorkflowExecutionResult: return json_format.MessageToDict(self.execution_output.payload) +@dataclass +class Template: + """Template dataclass. + + Store resolved template data. + It keeps also ResourceResolution object to call `store_template` method. + """ + + resource_resolution: "ResourceResolution" = field(repr=False) + blueprint_name: str + blueprint_version: str + artifact_name: str = None + result: str = None + resolution_key: str = None + resource_type: str = None + resource_id: str = None + + def store(self) -> None: + """Store template using blueprintprocessor HTTP API. + + It uses ResourceResolution `store_template` method. + """ + self.resource_resolution.store_template( + blueprint_name=self.blueprint_name, + blueprint_version=self.blueprint_version, + artifact_name=self.artifact_name, + result=self.result, + resolution_key=self.resolution_key, + resource_type=self.resource_type, + resource_id=self.resource_id, + ) + + class ResourceResolution: """Resource resolution class. @@ -191,20 +228,26 @@ class ResourceResolution: self, *, server_address: str = "127.0.0.1", - server_port: int = "9111", + # GRPC client configuration + grpc_server_port: int = 9111, use_ssl: bool = False, root_certificates: bytes = None, private_key: bytes = None, certificate_chain: bytes = None, - # Authentication header configuration + # Authentication header configuration for GRPC client use_header_auth: bool = False, header_auth_token: str = None, + # HTTP client configuration + http_server_port: int = 8080, + http_auth_user: str = None, + http_auth_pass: str = None, + http_use_ssl: bool = True, ) -> None: """Resource resolution object initialization. Args: server_address (str, optional): gRPC server address. Defaults to "127.0.0.1". - server_port (int, optional): gRPC server address port. Defaults to "9111". + grpc_server_port (int, optional): gRPC server address port. Defaults to 9111. use_ssl (bool, optional): Boolean flag to determine if secure channel should be created or not. Defaults to False. root_certificates (bytes, optional): The PEM-encoded root certificates. None if it shouldn't be used. @@ -216,33 +259,49 @@ class ResourceResolution: use_header_auth (bool, optional): Boolean flag to determine if authorization headed shoud be added for every call or not. Defaults to False. header_auth_token (str, optional): Authorization token value. Defaults to None. + If no value is provided "AUTH_TOKEN" environment variable will be used. + http_server_port (int, optional): HTTP server address port. Defaults to 8080. + http_auth_user (str, optional): Username used for HTTP requests authorization. Defaults to None. + If no value is provided "API_USERNAME" environment variable will be used. + http_auth_pass (str, optional): Password used for HTTP requests authorization. Defaults to None. + If no value is provided "API_PASSWORD" environment variable will be used. + http_use_ssl (bool, optional): Determines if secure connection should be used for HTTP requests. + Defaults to False. """ # Logger self.logger: Logger = getLogger(__name__) - # Client settings - self.client_server_address: str = server_address - self.client_server_port: str = server_port - self.client_use_ssl: bool = use_ssl - self.client_root_certificates: bytes = root_certificates - self.client_private_key: bytes = private_key - self.client_certificate_chain: bytes = certificate_chain - self.client_use_header_auth: bool = use_header_auth - self.client_header_auth_token: str = header_auth_token - self.client: Client = None + # GrpcClient settings + self.grpc_client_server_address: str = server_address + self.grpc_client_server_port: str = grpc_server_port + self.grpc_client_use_ssl: bool = use_ssl + self.grpc_client_root_certificates: bytes = root_certificates + self.grpc_client_private_key: bytes = private_key + self.grpc_client_certificate_chain: bytes = certificate_chain + self.grpc_client_use_header_auth: bool = use_header_auth + self.grpc_client_header_auth_token: str = header_auth_token or getenv("AUTH_TOKEN") + self.grpc_client: GrpcClient = None + # HttpClient settings + self.http_client: HttpClient = HttpClient( + server_address, + server_port=http_server_port, + auth_user=http_auth_user or getenv("API_USERNAME"), + auth_pass=http_auth_pass or getenv("API_PASSWORD"), + use_ssl=http_use_ssl, + ) def __enter__(self) -> "ResourceResolution": """Enter ResourceResolution instance context. - Client connection is created. + GrpcClient connection is created. """ - self.client = Client( - server_address=f"{self.client_server_address}:{self.client_server_port}", - use_ssl=self.client_use_ssl, - root_certificates=self.client_root_certificates, - private_key=self.client_private_key, - certificate_chain=self.client_certificate_chain, - use_header_auth=self.client_use_header_auth, - header_auth_token=self.client_header_auth_token, + self.grpc_client = GrpcClient( + server_address=f"{self.grpc_client_server_address}:{self.grpc_client_server_port}", + use_ssl=self.grpc_client_use_ssl, + root_certificates=self.grpc_client_root_certificates, + private_key=self.grpc_client_private_key, + certificate_chain=self.grpc_client_certificate_chain, + use_header_auth=self.grpc_client_use_header_auth, + header_auth_token=self.grpc_client_header_auth_token, ) return self @@ -254,9 +313,9 @@ class ResourceResolution: ) -> None: """Exit ResourceResolution instance context. - Client connection is closed. + GrpcClient connection is closed. """ - self.client.close() + self.grpc_client.close() def execute_workflows(self, *workflows: WorkflowExecution) -> Generator[WorkflowExecutionResult, None, None]: """Execute provided workflows. @@ -270,7 +329,7 @@ class ResourceResolution: AttributeError: Raises if client object is not created. It occurs only if you not uses context manager. Then user have to create client instance for ResourceResolution object by himself calling: ``` - resource_resoulution.client = Client( + resource_resoulution.client = GrpcClient( server_address=f"{resource_resoulution.client_server_address}:{resource_resoulution.client_server_port}", use_ssl=resource_resoulution.client_use_ssl, root_certificates=resource_resoulution.client_root_certificates, @@ -287,8 +346,112 @@ class ResourceResolution: with both WorkflowExection object and server response for it's request. """ self.logger.debug("Execute workflows") - if not self.client: + if not self.grpc_client: raise AttributeError("gRPC client not connected") - for response, workflow in zip(self.client.process((workflow.message for workflow in workflows)), workflows): + for response, workflow in zip( + self.grpc_client.process((workflow.message for workflow in workflows)), workflows + ): yield WorkflowExecutionResult(workflow, response) + + def _check_template_resolve_params( + self, resolution_key: str = None, resource_type: str = None, resource_id: str = None + ): + """Check template API request parameters. + + It's possible to store/retrieve templates using pair of artifact name and resolution key OR + resource type and resource id. This method checks if valid combination of parameters were used. + + Args: + resolution_key (str, optional): resolutionKey HTTP request parameter value. Defaults to None. + resource_type (str, optional): resourceType HTTP request parameter value. Defaults to None. + resource_id (str, optional): resourceId HTTP request parameter value. Defaults to None. + + Raises: + AttributeError: Invalid combination of parametes used + """ + if not any([resolution_key, all([resource_type, resource_id])]): + raise AttributeError( + "To store/retrieve template resolution_key and artifact_name or both resource_type and resource_id have to be provided" + ) + + def store_template( + self, + blueprint_name: str, + blueprint_version: str, + result: str, + artifact_name: str, + resolution_key: str = None, + resource_type: str = None, + resource_id: str = None, + ) -> None: + """Store template using blueprintprocessor HTTP API. + + Prepare and send a request to store resolved template using blueprint name, blueprint version + and pair of artifact name and resolution key OR resource type and resource id. + + Method returns Template dataclass, which stores all template data and can be used to update + it's result. + + Args: + blueprint_name (str): Blueprint name + blueprint_version (str): Blueprint version + result (str): Template result + artifact_name (str): Artifact name + resolution_key (str, optional): Resolution key. Defaults to None. + resource_type (str, optional): Resource type. Defaults to None. + resource_id (str, optional): Resource ID. Defaults to None. + """ + self.logger.debug("Store template") + self._check_template_resolve_params(resolution_key, resource_type, resource_id) + base_endpoint: str = f"template/{blueprint_name}/{blueprint_version}" + if resolution_key and artifact_name: + endpoint: str = f"{base_endpoint}/{artifact_name}/{resolution_key}" + else: + endpoint: str = f"{base_endpoint}/{resource_type}/{resource_id}" + response = self.http_client.send_request( + "POST", endpoint, headers={"Content-Type": "application/json"}, data=json.dumps({"result": result}) + ) + + def retrieve_template( + self, + blueprint_name: str, + blueprint_version: str, + artifact_name: str, + resolution_key: str = None, + resource_type: str = None, + resource_id: str = None, + ) -> Template: + """Get stored template using blueprintprocessor's HTTP API. + + Prepare and send a request to retrieve resolved template using blueprint name, blueprint version + and pair of artifact name and resolution key OR resource type and resource id. + + Args: + blueprint_name (str): Blueprint name + blueprint_version (str): Blueprint version + artifact_name (str): Artifact name + resolution_key (str, optional): Resolution key. Defaults to None. + resource_type (str, optional): Resource type. Defaults to None. + resource_id (str, optional): Resource ID. Defaults to None. + """ + self.logger.debug("Retrieve template") + self._check_template_resolve_params(resolution_key, resource_type, resource_id) + params: dict = {"bpName": blueprint_name, "bpVersion": blueprint_version, "artifactName": artifact_name} + if resolution_key: + params.update({"resolutionKey": resolution_key}) + else: + params.update({"resourceType": resource_type, "resourceId": resource_id}) + response = self.http_client.send_request( + "GET", "template", headers={"Accept": "application/json"}, params=params + ) + return Template( + resource_resolution=self, + blueprint_name=blueprint_name, + blueprint_version=blueprint_version, + artifact_name=artifact_name, + resolution_key=resolution_key, + resource_type=resource_type, + resource_id=resource_id, + result=response.json()["result"], + ) diff --git a/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py b/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py index 4b03f0b36..734059f3d 100644 --- a/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py +++ b/ms/py-executor/resource_resolution/tests/authorization_interceptor_test.py @@ -17,7 +17,7 @@ from unittest.mock import MagicMock, _Call import pytest -from resource_resolution.authorization import AuthTokenInterceptor, NewClientCallDetails +from resource_resolution.grpc.authorization import AuthTokenInterceptor, NewClientCallDetails def test_resource_resolution_auth_token_interceptor(): diff --git a/ms/py-executor/resource_resolution/tests/client_test.py b/ms/py-executor/resource_resolution/tests/client_test.py deleted file mode 100644 index 2b94220f6..000000000 --- a/ms/py-executor/resource_resolution/tests/client_test.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Copyright 2019 Deutsche Telekom. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -from unittest.mock import MagicMock, patch - -from resource_resolution.client import Client - - -@patch("resource_resolution.client.insecure_channel") -def test_resource_resoulution_insecure_channel(insecure_channel_mock: MagicMock): - """Test if insecure_channel connection is called.""" - with patch.object(Client, "close") as client_close_method_mock: # Type MagicMock - with Client("127.0.0.1:3333"): - pass - insecure_channel_mock.called_once_with() - client_close_method_mock.called_once_with() diff --git a/ms/py-executor/resource_resolution/tests/grpc_client_test.py b/ms/py-executor/resource_resolution/tests/grpc_client_test.py new file mode 100644 index 000000000..8217b0f25 --- /dev/null +++ b/ms/py-executor/resource_resolution/tests/grpc_client_test.py @@ -0,0 +1,28 @@ +"""Copyright 2019 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from unittest.mock import MagicMock, patch + +from resource_resolution.grpc.client import Client + + +@patch("resource_resolution.grpc.client.insecure_channel") +def test_resource_resoulution_insecure_channel(insecure_channel_mock: MagicMock): + """Test if insecure_channel connection is called.""" + with patch.object(Client, "close") as client_close_method_mock: # Type MagicMock + with Client("127.0.0.1:3333"): + pass + insecure_channel_mock.called_once_with() + client_close_method_mock.called_once_with() diff --git a/ms/py-executor/resource_resolution/tests/http_client_test.py b/ms/py-executor/resource_resolution/tests/http_client_test.py new file mode 100644 index 000000000..2279fde7a --- /dev/null +++ b/ms/py-executor/resource_resolution/tests/http_client_test.py @@ -0,0 +1,38 @@ +"""Copyright 2020 Deutsche Telekom. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from unittest.mock import MagicMock, patch + +from resource_resolution.http.client import Client + + +@patch("resource_resolution.http.client.request") +def test_http_client(request_mock): + c = Client("127.0.0.1", 8080) + assert c.auth is None + c = Client("127.0.0.1", 8080, auth_user="user") + assert c.auth is None + c = Client("127.0.0.1", 8080, auth_user="user", auth_pass="pass") + assert c.auth == ("user", "pass") + + assert c.protocol == "http" + assert c.url == "http://127.0.0.1:8080/api/v1" + + c = Client("127.0.0.1", 8081, use_ssl=True) + assert c.protocol == "https" + assert c.url == "https://127.0.0.1:8081/api/v1" + + c.send_request("GET", "something") + request_mock.assert_called_once_with(method="GET", url=f"{c.url}/something", verify=False, auth=None) diff --git a/ms/py-executor/resource_resolution/tests/resource_resolution_test.py b/ms/py-executor/resource_resolution/tests/resource_resolution_test.py index 8a41357e6..274802279 100644 --- a/ms/py-executor/resource_resolution/tests/resource_resolution_test.py +++ b/ms/py-executor/resource_resolution/tests/resource_resolution_test.py @@ -13,12 +13,17 @@ See the License for the specific language governing permissions and limitations under the License. """ +import json +from unittest.mock import patch, MagicMock + from google.protobuf import json_format from pytest import raises from resource_resolution.resource_resolution import ( ExecutionServiceInput, ExecutionServiceOutput, + ResourceResolution, + Template, WorkflowExecution, WorkflowExecutionResult, WorkflowMode, @@ -103,3 +108,62 @@ def test_workflow_execution_result_class(): execution_output.status.code = 500 assert execution_result.has_error assert execution_result.error_message == "" + + +def test_resource_resolution_check_resolve_params(): + """Check values of potentially HTTP parameters.""" + rr = ResourceResolution() + with raises(AttributeError): + rr._check_template_resolve_params() + rr._check_template_resolve_params(resource_type="test") + rr._check_template_resolve_params(resource_id="test") + rr._check_template_resolve_params(resolution_key="test") + rr._check_template_resolve_params(resource_type="test", resource_id="test") + + +def test_store_template(): + """Test store_template method. + + Checks if http_client send_request method is called with valid parameters. + """ + rr = ResourceResolution(server_address="127.0.0.1", http_server_port=8080) + rr.http_client = MagicMock() + rr.store_template( + blueprint_name="test_blueprint_name", + blueprint_version="test_blueprint_version", + artifact_name="test_artifact_name", + resolution_key="test_resolution_key", + result="test_result", + ) + rr.http_client.send_request.assert_called_once_with( + "POST", + "template/test_blueprint_name/test_blueprint_version/test_artifact_name/test_resolution_key", + data=json.dumps({"result": "test_result"}), + headers={"Content-Type": "application/json"}, + ) + + +def test_retrieve_template(): + """Test retrieve_template method. + + Checks if http_client send_request method is called with valid parameters. + """ + rr = ResourceResolution(server_address="127.0.0.1", http_server_port=8080) + rr.http_client = MagicMock() + rr.retrieve_template( + blueprint_name="test_blueprint_name", + blueprint_version="test_blueprint_version", + artifact_name="test_artifact_name", + resolution_key="test_resolution_key", + ) + rr.http_client.send_request.assert_called_once_with( + "GET", + "template", + params={ + "bpName": "test_blueprint_name", + "bpVersion": "test_blueprint_version", + "artifactName": "test_artifact_name", + "resolutionKey": "test_resolution_key", + }, + headers={"Accept": "application/json"}, + ) -- cgit 1.2.3-korg