Merge "Refactor DMaaP simulator and add tests."

14 files changed, 540 insertions, 0 deletions

diff --git a/tests/dcaegen2/testcases/resources/robot_library/CertsLibrary.py b/tests/dcaegen2/testcases/resources/robot_library/CertsLibrary.py
new file mode 100644
index 00000000..b8189422
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/CertsLibrary.py
@@ -0,0 +1,42 @@
+
+import os
+import subprocess
+import time
+
+from robot.api import logger
+
+
+class CertsLibrary(object):
+
+    def __init__(self):
+        pass
+
+
+
+    @staticmethod
+    def generate_certs():
+        ws = os.environ['WORKSPACE']
+        script2run = ws + "/tests/dcaegen2/testcases/resources/gen-certs.sh"
+        logger.info("Running script: " + script2run)
+        logger.console("Running script: " + script2run)
+        subprocess.call([script2run, ws])
+        time.sleep(5)
+        return
+
+    @staticmethod
+    def remove_certs():
+        ws = os.environ['WORKSPACE']
+        script2run = ws + "/tests/dcaegen2/testcases/resources/rm-certs.sh"
+        logger.info("Running script: " + script2run)
+        logger.console("Running script: " + script2run)
+        subprocess.call([script2run, ws])
+        time.sleep(5)
+        return
+
+
+
+
+
+
+
+
diff --git a/tests/dcaegen2/testcases/resources/robot_library/DcaeLibrary.py b/tests/dcaegen2/testcases/resources/robot_library/DcaeLibrary.py
new file mode 100644
index 00000000..a467431f
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/DcaeLibrary.py
@@ -0,0 +1,112 @@
+'''
+Created on Aug 18, 2017
+
+@author: sw6830
+'''
+from robot.api import logger
+import uuid
+import time
+import datetime
+import json
+import os
+import platform
+import subprocess
+import paramiko
+
+
+class DcaeLibrary(object):
+
+    def __init__(self):
+        pass
+    
+    @staticmethod
+    def enable_vesc_with_cert_basic_auth():
+        global client
+        if 'Windows' in platform.system():
+            try:
+                DcaeLibrary.enable_https_auth_for_windows_platform_system()
+            finally:
+                client.close()
+            return
+        DcaeLibrary.enable_https_auth_for_non_windows_platform_system()
+        return
+
+    @staticmethod
+    def enable_https_auth_for_non_windows_platform_system():
+        ws = os.environ['WORKSPACE']
+        script2run = ws + "/tests/dcaegen2/testcases/resources/vesc_enable_https_auth.sh"
+        logger.info("Running script: " + script2run)
+        logger.console("Running script: " + script2run)
+        subprocess.call(script2run)
+        time.sleep(5)
+
+    @staticmethod
+    def enable_https_auth_for_windows_platform_system():
+        global client
+        client = paramiko.SSHClient()
+        client.load_system_host_keys()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        client.connect(os.environ['CSIT_IP'], port=22, username=os.environ['CSIT_USER'], password=os.environ['CSIT_PD'])
+        stdin, stdout, stderr = client.exec_command(
+            '%{WORKSPACE}/tests/dcaegen2/testcases/resources/vesc_enable_https_auth.sh')
+        logger.console(stdout.read())
+
+    @staticmethod
+    def is_json_empty(resp):
+        logger.info("Enter is_json_empty: resp.text: " + resp.text)
+        if resp.text is None or len(resp.text) < 2:
+            return 'True'
+        return 'False'
+    
+    @staticmethod
+    def generate_uuid():
+        """generate a uuid"""
+        return uuid.uuid4()
+    
+    @staticmethod
+    def get_json_value_list(jsonstr, keyval):
+        logger.info("Enter Get_Json_Key_Value_List")
+        if jsonstr is None or len(jsonstr) < 2:
+            logger.info("No Json data found")
+            return []
+        try:
+            return DcaeLibrary.extract_list_of_items_from_json_string(jsonstr, keyval)
+        except Exception as e:
+            logger.info("Json data parsing fails")
+            print str(e)
+            return []
+
+    @staticmethod
+    def extract_list_of_items_from_json_string(jsonstr, keyval):
+        data = json.loads(jsonstr)
+        nodelist = []
+        for item in data:
+            nodelist.append(item[keyval])
+        return nodelist
+
+    @staticmethod
+    def generate_millitimestamp_uuid():
+        """generate a millisecond timestamp uuid"""
+        then = datetime.datetime.now()
+        return int(time.mktime(then.timetuple())*1e3 + then.microsecond/1e3)
+    
+    @staticmethod
+    def test():
+        import json
+        from pprint import pprint
+
+        with open('robot/assets/dcae/ves_volte_single_fault_event.json') as data_file:    
+            data = json.load(data_file)
+
+        data['event']['commonEventHeader']['version'] = '5.0'
+        pprint(data)
+
+
+if __name__ == '__main__':
+
+    lib = DcaeLibrary()
+    lib.enable_vesc_https_auth()
+    
+    ret = lib.setup_dmaap_server()
+    print ret
+    time.sleep(100000)
diff --git a/tests/dcaegen2/testcases/resources/robot_library/DcaeVariables.py b/tests/dcaegen2/testcases/resources/robot_library/DcaeVariables.py
new file mode 100644
index 00000000..47d169f1
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/DcaeVariables.py
@@ -0,0 +1,13 @@
+import os
+
+
+def get_environment_variable(env_varstr):
+    return os.environ.get(env_varstr)
+
+
+DCAE_HEALTH_CHECK_URL = "http://135.205.228.129:8500"
+DCAE_HEALTH_CHECK_URL1 = "http://135.205.228.170:8500"
+
+CommonEventSchema = get_environment_variable('WORKSPACE') + "/tests/dcaegen2/testcases/assets/json_events/CommonEventFormat_30.2_ONAP.json"
+
+IsRobotRun = False
diff --git a/tests/dcaegen2/testcases/resources/robot_library/DmaapLibrary.py b/tests/dcaegen2/testcases/resources/robot_library/DmaapLibrary.py
new file mode 100644
index 00000000..c9a0ff7b
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/DmaapLibrary.py
@@ -0,0 +1,79 @@
+from Queue import Queue
+
+import robot.api.logger as logger
+import threading
+import time
+
+import DcaeVariables
+from robot_library.dmaap_simulator import DMaaPServer
+from robot_library.dmaap_simulator.DMaaPQueue import DMaaPQueue
+
+class DmaapLibrary(object):
+
+    dmaap_queue = None
+    dmaap_server = None
+    server_thread = None
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def setup_dmaap_server(port_num=3904):
+        try:
+            DmaapLibrary.start_dmaap_server_on_new_thread(port_num)
+            return "true"
+        except Exception as e:
+            print (str(e))
+            return "false"
+
+    @staticmethod
+    def start_dmaap_server_on_new_thread(port_num):
+        DmaapLibrary.dmaap_queue = DMaaPQueue(Queue())
+        DmaapLibrary.dmaap_server = DMaaPServer.create_dmaap_server(DmaapLibrary.dmaap_queue, port=port_num)
+        DmaapLibrary.server_thread = threading.Thread(name='DMAAP_HTTPServer',
+                                                      target=DmaapLibrary.dmaap_server.serve_forever)
+        DmaapLibrary.server_thread.start()
+        logger.console("DMaaP Mockup Sever started")
+        DcaeVariables.IsRobotRun = True
+        time.sleep(2)
+
+    @staticmethod
+    def shutdown_dmaap():
+        if DmaapLibrary.dmaap_server is not None:
+            DmaapLibrary.dmaap_server.shutdown()
+            logger.console("DMaaP Server shut down")
+            time.sleep(3)
+            return "true"
+        else:
+            return "false"
+
+    @staticmethod
+    def cleanup_ves_events():
+        if DmaapLibrary.server_thread is not None:
+            DmaapLibrary.dmaap_queue.clean_up_event()
+            logger.console("DMaaP event queue is cleaned up")
+            return "true"
+        logger.console("DMaaP server not started yet")
+        return "false"
+
+    @staticmethod
+    def dmaap_message_receive_on_topic(evtobj, topic):
+
+        evt_str = DmaapLibrary.dmaap_queue.deque_event()
+        while evt_str != None:
+            if evtobj in evt_str and topic in evt_str:
+                logger.info("DMaaP Receive Expected Publish Event:\n" + evt_str)
+                logger.info("On Expected Topic:\n" + topic)
+                return 'true'
+            evt_str = DmaapLibrary.dmaap_queue.deque_event()
+        return 'false'
+
+    @staticmethod
+    def dmaap_message_receive(evtobj):
+        evt_str = DmaapLibrary.dmaap_queue.deque_event()
+        while evt_str != None:
+            if evtobj in evt_str:
+                logger.info("DMaaP Receive Expected Publish Event:\n" + evt_str)
+                return 'true'
+            evt_str = DmaapLibrary.dmaap_queue.deque_event()
+        return 'false'
diff --git a/tests/dcaegen2/testcases/resources/robot_library/README.md b/tests/dcaegen2/testcases/resources/robot_library/README.md
new file mode 100644
index 00000000..ac432f79
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/README.md
@@ -0,0 +1,14 @@
+# Robot Library
+This catalog contains python files used in Robot tests for dcaegen2.ves.
+
+# DMaaP Simulator
+Catalog dmaap_simulator contains python implementation of DMaaP simulator. It uses python BaseHTTPServer to expose endpoints.
+
+# DMaaP Tests
+Catalog dmaap_test contains tests that are used to validate DMaaP simulator. Test are using "pytest" and "MagicMock". 
+
+### In order to run tests: 
+1. create virtual environemnt with Python 2.7;
+2. install requirements from file requirements.txt located in dmaap_test;
+3. set environement variable WORKSPACE to point root csit catalog
+4. run py.test command in catalog dmaap_test
diff --git a/tests/dcaegen2/testcases/resources/robot_library/__init__.py b/tests/dcaegen2/testcases/resources/robot_library/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/__init__.py
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPHandler.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPHandler.py
new file mode 100644
index 00000000..f1c46e19
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPHandler.py
@@ -0,0 +1,73 @@
+'''
+Created on Aug 15, 2017
+
+@author: sw6830
+'''
+import os
+import posixpath
+import BaseHTTPServer
+import urllib
+import urlparse
+import cgi
+import sys
+import shutil
+import mimetypes
+from robot_library import DcaeVariables
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+class DMaaPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+
+    def __init__(self, dmaap_simulator, *args):
+        self.dmaap_simulator = dmaap_simulator
+        BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args)
+
+    def do_POST(self):
+        if 'POST' not in self.requestline:
+            resp_code = 405
+        else:
+            resp_code = self.parse_the_posted_data()
+
+        if resp_code == 0:
+            self.send_successful_response()
+        else:
+            self.send_response(resp_code)
+
+    def parse_the_posted_data(self):
+        topic = self.extract_topic_from_path()
+        content_len = self.get_content_length()
+        post_body = self.rfile.read(content_len)
+        post_body = self.get_json_part_of_post_body(post_body)
+        event = "{\"" + topic + "\":" + post_body + "}"
+        if self.dmaap_simulator.enque_event(event):
+            resp_code = 0
+        else:
+            print "enque event fails"
+            resp_code = 500
+        return resp_code
+
+    def get_json_part_of_post_body(self, post_body):
+        indx = post_body.index("{")
+        if indx != 0:
+            post_body = post_body[indx:]
+        return post_body
+
+    def extract_topic_from_path(self):
+        return self.path["/events/".__len__():]
+
+    def get_content_length(self):
+        return int(self.headers.getheader('content-length', 0))
+
+    def send_successful_response(self):
+        if 'clientThrottlingState' in self.requestline:
+            self.send_response(204)
+        else:
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write("{'count': 1, 'serverTimeMs': 3}")
+            self.wfile.close()
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPQueue.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPQueue.py
new file mode 100644
index 00000000..3d3a81fa
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPQueue.py
@@ -0,0 +1,48 @@
+class DMaaPQueue(object):
+
+    def __init__(self, event_queue, wait_timeout_sec=25):
+        self.event_queue = event_queue
+        self.wait_timeout_sec = wait_timeout_sec
+
+    def set_deque_event_timeout(self, wait_timeout_sec):
+        self.wait_timeout_sec = wait_timeout_sec
+
+    def clean_up_event(self):
+        if self.queue_is_valid():
+            with self.event_queue.mutex:
+                try:
+                    self.event_queue.queue.clear()
+                except:
+                    pass
+
+    def enque_event(self, event):
+        event_placed_on_queue = False
+        if self.queue_is_valid():
+            event_placed_on_queue = self._enque_event(event, event_placed_on_queue)
+        return event_placed_on_queue
+
+    def _enque_event(self, event, event_placed_on_queue):
+        try:
+            self.event_queue.put(event)
+            event_placed_on_queue = True
+        except Exception as e:
+            print (str(e))
+        return event_placed_on_queue
+
+    def deque_event(self, wait_sec=None):
+        if wait_sec is None:
+            wait_sec = self.wait_timeout_sec
+        event_from_queue = None
+        if self.queue_is_valid():
+            event_from_queue = self._deque_event(event_from_queue, wait_sec)
+        return event_from_queue
+
+    def _deque_event(self, event_from_queue, wait_sec):
+        try:
+            event_from_queue = self.event_queue.get(True, wait_sec)
+        except Exception as e:
+            print("DMaaP Event dequeue timeout")
+        return event_from_queue
+
+    def queue_is_valid(self):
+        return self.event_queue is not None
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPServer.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPServer.py
new file mode 100644
index 00000000..37499be8
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/DMaaPServer.py
@@ -0,0 +1,22 @@
+import BaseHTTPServer
+import DMaaPHandler
+
+
+class DMaaPServer(BaseHTTPServer.HTTPServer):
+
+    def __init__(self, server_address, protocol, dmaap_simulator):
+
+        def handler_class_constructor(*args):
+            DMaaPHandler.DMaaPHandler(dmaap_simulator, *args)
+        DMaaPHandler.protocol_version = protocol
+        BaseHTTPServer.HTTPServer.__init__(self, server_address, handler_class_constructor)
+
+        serer_address = self.socket.getsockname()
+        print "Serving HTTP on", serer_address[0], "port", serer_address[1], "..."
+
+
+def create_dmaap_server(dmaap_simulator, protocol="HTTP/1.0", port=3904):
+    server_address = ('', port)
+    httpd = DMaaPServer(server_address, protocol, dmaap_simulator)
+
+    return httpd
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/__init__.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_simulator/__init__.py
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/__init__.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/__init__.py
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/requirements.txt b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/requirements.txt
new file mode 100644
index 00000000..7a687204
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/requirements.txt
@@ -0,0 +1,23 @@
+atomicwrites==1.4.0
+attrs==19.3.0
+backports.functools-lru-cache==1.6.1
+configparser==4.0.2
+contextlib2==0.6.0.post1
+funcsigs==1.0.2
+httplib2==0.18.1
+importlib-metadata==1.7.0
+magicmock==0.3
+mock==3.0.5
+mocker==1.1.1
+more-itertools==5.0.0
+packaging==20.4
+pathlib2==2.3.5
+pluggy==0.13.1
+py==1.9.0
+pyparsing==2.4.7
+pytest==4.6.11
+pytest-mock==2.0.0
+scandir==1.10.0
+six==1.15.0
+wcwidth==0.2.5
+zipp==1.2.0
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSQueue.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSQueue.py
new file mode 100644
index 00000000..f278a391
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSQueue.py
@@ -0,0 +1,52 @@
+from Queue import Queue
+import pytest
+from robot_library.dmaap_simulator.DMaaPQueue import DMaaPQueue
+
+wait_sec_for_dequeing_event = 0.1
+test_event = "\"topic\":{\"test\":123}"
+
+
+class TestDMaaPQueue:
+
+    dmaap_simulator = None
+
+    @pytest.fixture(autouse=True, scope="function")
+    def initiate_dmaap_simulator(self):
+        TestDMaaPQueue.dmaap_simulator = DMaaPQueue(Queue())
+        TestDMaaPQueue.dmaap_simulator.set_deque_event_timeout(wait_sec_for_dequeing_event)
+        yield
+
+    def test_when_queue_is_empty_then_deque_returns_none(self):
+        # when
+        event = TestDMaaPQueue.dmaap_simulator.deque_event()
+
+        # then
+        assert event is None
+
+    def test_when_enque_event_then_dequeue_return_same_event(self):
+        # when
+        TestDMaaPQueue.dmaap_simulator.enque_event(test_event)
+        event = TestDMaaPQueue.dmaap_simulator.deque_event()
+
+        # then
+        assert event == test_event
+
+    def test_when_enque_and_dequeue_event_then_deque_return_none(self):
+        # when
+        TestDMaaPQueue.dmaap_simulator.enque_event(test_event)
+        TestDMaaPQueue.dmaap_simulator.deque_event()
+        event = TestDMaaPQueue.dmaap_simulator.deque_event()
+
+        # then
+        assert event is None
+
+    def test_when_enque_few_events_and_clean_up_then_dequeu_return_none(self):
+        # when
+        TestDMaaPQueue.dmaap_simulator.enque_event(test_event)
+        TestDMaaPQueue.dmaap_simulator.enque_event(test_event)
+        TestDMaaPQueue.dmaap_simulator.enque_event(test_event)
+        TestDMaaPQueue.dmaap_simulator.clean_up_event()
+        event = TestDMaaPQueue.dmaap_simulator.deque_event()
+
+        # then
+        assert event is None
diff --git a/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSimulator.py b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSimulator.py
new file mode 100644
index 00000000..82f95ff8
--- /dev/null
+++ b/tests/dcaegen2/testcases/resources/robot_library/dmaap_test/test_DMaaPSimulator.py
@@ -0,0 +1,62 @@
+import sys
+import pytest
+from mock import MagicMock
+
+sys.modules['robot'] = MagicMock()
+sys.modules['robot.api'] = MagicMock()
+sys.modules['robot.api.logger'] = MagicMock()
+from robot_library.DmaapLibrary import DmaapLibrary
+
+wait_sec_for_dequeing_event = 0.1
+test_event = "{\"test\":\"123\"}"
+test_topic = "topic"
+test_message = "\"" + test_topic + "\":" + test_event
+
+
+class TestDMaaPSimulator:
+
+    @pytest.fixture(autouse=True, scope="class")
+    def initiate_dmaap_simulator(self):
+        DmaapLibrary.setup_dmaap_server()
+        DmaapLibrary.dmaap_queue.set_deque_event_timeout(wait_sec_for_dequeing_event)
+        yield
+        assert DmaapLibrary.shutdown_dmaap() == "true"
+
+    @pytest.fixture(autouse=True, scope="function")
+    def clear_dmaap_simulator(self):
+        yield
+        DmaapLibrary.cleanup_ves_events()
+
+    def test_start_stop_dmaap_server(self):
+        # when / then
+        assert DmaapLibrary.dmaap_queue is not None
+        assert DmaapLibrary.dmaap_server is not None
+        assert DmaapLibrary.server_thread is not None
+
+    def test_dmaap_server_returns_true_when_event_is_present_on_queue(self):
+        # when
+        DmaapLibrary.dmaap_queue.enque_event(test_message)
+
+        # then
+        assert DmaapLibrary.dmaap_message_receive(test_event) == 'true'
+
+    def test_dmaap_server_returns_true_when_event_is_present_on_given_topic_on_queue(self):
+        # when
+        DmaapLibrary.dmaap_queue.enque_event(test_message)
+
+        # then
+        assert DmaapLibrary.dmaap_message_receive_on_topic(test_event, test_topic) == 'true'
+
+    def test_dmaap_server_returns_timeout_when_event_is_not_present_on_queue(self):
+        # when / then
+        assert DmaapLibrary.dmaap_message_receive(test_event) == 'false'
+
+    def test_dmaap_server_returns_false_when_queue_was_cleared(self):
+        # when
+        DmaapLibrary.dmaap_queue.enque_event(test_message)
+        DmaapLibrary.dmaap_queue.enque_event(test_message)
+        DmaapLibrary.dmaap_queue.enque_event(test_message)
+        DmaapLibrary.cleanup_ves_events()
+
+        # then
+        assert DmaapLibrary.dmaap_message_receive_on_topic(test_event, test_topic) == 'false'