Add Kafka support

- send changes on Kafka topic
- add endpoint for fetching changes from Kafka

Signed-off-by: Bogumil Zebek <bogumil.zebek@nokia.com>
Issue-ID: INT-1869
Signed-off-by: Zebek Bogumil <bogumil.zebek@nokia.com>
Change-Id: I349fdc4295659fc69407b5b1138281e2673f7938

3 files changed, 169 insertions, 3 deletions

diff --git a/src/python/tests/unit/test_netconf_change_listener.py b/src/python/tests/unit/test_netconf_change_listener.py
new file mode 100644
index 0000000..c9e11d3
--- /dev/null
+++ b/src/python/tests/unit/test_netconf_change_listener.py
@@ -0,0 +1,74 @@
+###
+# ============LICENSE_START=======================================================
+# Netconf Server
+# ================================================================================
+# Copyright (C) 2021 Nokia. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=========================================================
+###
+import unittest
+from unittest.mock import MagicMock
+
+import sys
+
+# we need to mock sysrepo library. It is not possible to install it in the newest version of the Linux distribution
+from netconf_server.sysrepo_interface.config_change_data import ConfigChangeData
+
+sys.modules['sysrepo'] = MagicMock()
+from netconf_server.netconf_change_listener import NetconfChangeListener
+
+KAFKA_TOPIC = "config"
+
+
+class TestNetconfChangeListener(unittest.TestCase):
+    def test_should_subscribe_on_run(self):
+        # given
+        subscriber1 = MagicMock()
+        subscriber2 = MagicMock()
+        kafka_client = MagicMock()
+        session = MagicMock()
+        netconf_change_listener = NetconfChangeListener([subscriber1, subscriber2], kafka_client, KAFKA_TOPIC)
+
+        # when
+        netconf_change_listener.run(session)
+
+        # then
+        subscriber1.subscribe_on_model_change.assert_called_once()
+        self.assertEqual(subscriber1.callback_function, netconf_change_listener._on_module_configuration_change)
+        subscriber2.subscribe_on_model_change.assert_called_once()
+        self.assertEqual(subscriber2.callback_function, netconf_change_listener._on_module_configuration_change)
+
+    def test_should_send_two_changes_at_kafka(self):
+        # given
+        subscriber1 = MagicMock()
+        subscriber2 = MagicMock()
+        kafka_client = MagicMock()
+        netconf_change_listener = NetconfChangeListener([subscriber1, subscriber2], kafka_client, KAFKA_TOPIC)
+        NetconfChangeListener._create_kafka_message = lambda _: MagicMock()
+
+        # when
+        netconf_change_listener._on_module_configuration_change(
+            ConfigChangeData(
+                event="event",
+                req_id=1,
+                changes=[MagicMock(), MagicMock()]
+            )
+        )
+
+        # then
+        self.assertEqual(kafka_client.send.call_count, 2)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/python/tests/unit/test_netconf_chang_listener.py b/src/python/tests/unit/test_netconf_change_listener_factory.py
index c2889f1..113f96c 100644
--- a/src/python/tests/unit/test_netconf_chang_listener.py
+++ b/src/python/tests/unit/test_netconf_change_listener_factory.py
@@ -20,16 +20,23 @@
 import unittest
 from unittest.mock import MagicMock
 
+import sys
+
+# we need to mock sysrepo library. It is not possible to install it in the newest version of the Linux distribution
+sys.modules['sysrepo'] = MagicMock()
+
+from netconf_server.netconf_app_configuration import NetconfAppConfiguration
 from netconf_server.netconf_change_listener_factory import NetconfChangeListenerFactory
 from tests.mocs.mocked_session import MockedSession
 
 
-class TestNetconfServer(unittest.TestCase):
+class TestNetconfChangeListenerFactory(unittest.TestCase):
 
     def test_should_create_and_run_netconf_server_with_one_model(self):
         # given
         modules_to_subscribe_names = ["test"]
-        server = NetconfChangeListenerFactory(modules_to_subscribe_names).create()
+        factory = TestNetconfChangeListenerFactory._given_netconf_change_listener_factory(modules_to_subscribe_names)
+        server = factory.create()
         session = MockedSession()
         session.subscribe_module_change = MagicMock()
 
@@ -42,7 +49,8 @@ class TestNetconfServer(unittest.TestCase):
     def test_should_create_and_run_netconf_server_with_multiple_models(self):
         # given
         modules_to_subscribe_names = ["test", "test2", "test3"]
-        server = NetconfChangeListenerFactory(modules_to_subscribe_names).create()
+        factory = TestNetconfChangeListenerFactory._given_netconf_change_listener_factory(modules_to_subscribe_names)
+        server = factory.create()
         session = MockedSession()
         session.subscribe_module_change = MagicMock()
 
@@ -51,3 +59,12 @@ class TestNetconfServer(unittest.TestCase):
 
         # then
         self.assertEqual(session.subscribe_module_change.call_count, 3)
+
+    @staticmethod
+    def _given_netconf_change_listener_factory(modules_to_subscribe_names: list) -> NetconfChangeListenerFactory:
+        app_configuration, _ = NetconfAppConfiguration.get_configuration(
+            ["../models", "models-configuration.ini", "127.0.0.1", "9092",
+             "kafka1"])  # type: NetconfAppConfiguration, str
+        factory = NetconfChangeListenerFactory(modules_to_subscribe_names, app_configuration)
+        NetconfChangeListenerFactory._try_to_create_kafka_client = lambda host, port: MagicMock()
+        return factory
diff --git a/src/python/tests/unit/test_netconf_kafka_message_factory.py b/src/python/tests/unit/test_netconf_kafka_message_factory.py
new file mode 100644
index 0000000..b899bd9
--- /dev/null
+++ b/src/python/tests/unit/test_netconf_kafka_message_factory.py
@@ -0,0 +1,75 @@
+from collections import namedtuple
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+import sys
+
+# we need to mock sysrepo library. It is not possible to install it in the newest version of the Linux distribution
+sys.modules['sysrepo'] = MagicMock()
+
+from netconf_server.sysrepo_interface.sysrepo_message_model import SysrepoMessage
+
+from netconf_server.netconf_kafka_message_factory import NetconfKafkaMessageFactory
+
+SYSREPO_MESSAGE_MODEL = namedtuple('SM', ['value', 'xpath', 'prev_val'])
+
+
+class TestNetconfKafkaMessageFactory(TestCase):
+    def test_should_return_empty_dict_when_sysrepo_message_is_none(self):
+        # when
+        actual = NetconfKafkaMessageFactory.create(None)
+
+        # then
+        self.assertEqual({}, actual)
+
+    def test_should_prepare_message_for_sysrepo_message_with_status_change_created(self):
+        # given
+        s = SYSREPO_MESSAGE_MODEL(44, '/pnf-simulator:config/itemValue1', None)
+
+        sysrepo_message = SysrepoMessage(s)
+        sysrepo_message.is_modified = lambda: False
+        sysrepo_message.is_created = lambda: True
+
+        # when
+        actual = NetconfKafkaMessageFactory.create(sysrepo_message)
+
+        # then
+        self.assertEqual(
+            {'type': 'ChangeCreated', 'new': {'path': '/pnf-simulator:config/itemValue1', 'value': 44}},
+            actual
+        )
+
+    def test_should_prepare_message_for_sysrepo_message_with_status_change_modified_no_old_value(self):
+        # given
+        s = SYSREPO_MESSAGE_MODEL(45, '/pnf-simulator:config/itemValue1', None)
+
+        sysrepo_message = SysrepoMessage(s)
+        sysrepo_message.is_modified = lambda: True
+        sysrepo_message.is_created = lambda: False
+
+        # when
+        actual = NetconfKafkaMessageFactory.create(sysrepo_message)
+
+        # then
+        self.assertEqual(
+            {'type': 'ChangeModified', 'new': {'path': '/pnf-simulator:config/itemValue1', 'value': 45}},
+            actual
+        )
+
+    def test_should_prepare_message_for_sysrepo_message_with_status_change_modified_old_value_exists(self):
+        # given
+        s = SYSREPO_MESSAGE_MODEL(45, '/pnf-simulator:config/itemValue1', 44)
+
+        sysrepo_message = SysrepoMessage(s)
+        sysrepo_message.is_modified = lambda: True
+        sysrepo_message.is_created = lambda: False
+
+        # when
+        actual = NetconfKafkaMessageFactory.create(sysrepo_message)
+
+        # then
+        self.assertEqual(
+            {'type': 'ChangeModified', 'old': {'path': '/pnf-simulator:config/itemValue1', 'value': 44}, 'new': {'path': '/pnf-simulator:config/itemValue1', 'value': 45}},
+            actual
+        )
+