Merge "Adding DB Init and setup"

8 files changed, 347 insertions, 57 deletions

diff --git a/components/pm-subscription-handler/pmsh_service/mod/__init__.py b/components/pm-subscription-handler/pmsh_service/mod/__init__.py
index d2f6f2fd..f8b1c598 100644
--- a/components/pm-subscription-handler/pmsh_service/mod/__init__.py
+++ b/components/pm-subscription-handler/pmsh_service/mod/__init__.py
@@ -1,5 +1,5 @@
 # ============LICENSE_START===================================================
-#  Copyright (C) 2019 Nordix Foundation.
+#  Copyright (C) 2019-2020 Nordix Foundation.
 # ============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +15,49 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 # ============LICENSE_END=====================================================
+import os
+from urllib.parse import quote
 
+from connexion import App
+from flask_sqlalchemy import SQLAlchemy
 
-# empty __init__.py so that pytest can add correct path to coverage report,
-# -- per pytest best practice guideline
+import mod.pmsh_logging as logger
+
+db = SQLAlchemy()
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+
+def create_prod_app():
+    logger.create_loggers(os.getenv('LOGS_PATH'))
+    connex_app = App(__name__, specification_dir=basedir)
+    app = connex_app.app
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    app.config['SQLALCHEMY_RECORD_QUERIES'] = True
+    app.config['SQLALCHEMY_DATABASE_URI'] = get_db_connection_url()
+    db.init_app(app)
+    return app
+
+
+def create_test_app():
+    logger.create_loggers('./unit_test_logs')
+    connex_app = App(__name__, specification_dir=basedir)
+    app = connex_app.app
+    app.config['TESTING'] = True
+    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+    app.config['SQLALCHEMY_RECORD_QUERIES'] = True
+    app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('TEST_DB_URL', 'sqlite://')
+    db.init_app(app)
+    return app
+
+
+def get_db_connection_url():
+    pg_host = os.getenv('PMSH_PG_URL')
+    pg_user = os.getenv('PMSH_PG_USERNAME')
+    pg_user_pass = os.getenv('PMSH_PG_PASSWORD')
+    pmsh_db_name = os.getenv('PMSH_DB_NAME', 'pmsh')
+    pmsh_db_port = os.getenv('PMSH_PG_PORT', '5432')
+    db_url = f'postgres+psycopg2://{quote(str(pg_user), safe="")}:' \
+        f'{quote(str(pg_user_pass), safe="")}@{pg_host}:{pmsh_db_port}/{pmsh_db_name}'
+    if 'None' in db_url:
+        raise Exception(f'Invalid DB connection URL: {db_url} .. exiting app!')
+    return db_url
diff --git a/components/pm-subscription-handler/pmsh_service/mod/aai_client.py b/components/pm-subscription-handler/pmsh_service/mod/aai_client.py
index 8b51a712..747846f1 100755
--- a/components/pm-subscription-handler/pmsh_service/mod/aai_client.py
+++ b/components/pm-subscription-handler/pmsh_service/mod/aai_client.py
@@ -16,14 +16,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # ============LICENSE_END=====================================================
 import json
-import os
 import uuid
+from os import environ
 
 import requests
 from requests.auth import HTTPBasicAuth
 
 import mod.pmsh_logging as logger
-from mod.subscription import Subscription, XnfFilter
+from mod.network_function import NetworkFunction
+from mod.subscription import Subscription, NetworkFunctionFilter
 
 
 def get_pmsh_subscription_data(cbs_data):
@@ -34,28 +35,29 @@ def get_pmsh_subscription_data(cbs_data):
         cbs_data: json app config from the Config Binding Service.
 
     Returns:
-        Subscription, set(Xnf): `Subscription` <Subscription> object, set of XNFs to be added.
+        Subscription, set(NetworkFunctions): `Subscription` <Subscription> object,
+        set of NetworkFunctions to be added.
 
     Raises:
         RuntimeError: if AAI data cannot be retrieved.
     """
-    aai_xnf_data = _get_all_aai_xnf_data()
-    if aai_xnf_data:
+    aai_nf_data = _get_all_aai_nf_data()
+    if aai_nf_data:
         sub = Subscription(**cbs_data['policy']['subscription'])
-        xnfs = _filter_xnf_data(aai_xnf_data, XnfFilter(**sub.nfFilter))
+        nfs = _filter_nf_data(aai_nf_data, NetworkFunctionFilter(**sub.nfFilter))
     else:
         raise RuntimeError('Failed to get data from AAI')
-    return sub, xnfs
+    return sub, nfs
 
 
-def _get_all_aai_xnf_data():
+def _get_all_aai_nf_data():
     """
-    Return queried xnf data from the AAI service.
+    Return queried nf data from the AAI service.
 
     Returns:
         json: the json response from AAI query, else None.
     """
-    xnf_data = None
+    nf_data = None
     try:
         session = requests.Session()
         aai_endpoint = f'{_get_aai_service_url()}{"/aai/v16/query"}'
@@ -74,10 +76,10 @@ def _get_all_aai_xnf_data():
                                data=json_data, params=params, verify=False)
         response.raise_for_status()
         if response.ok:
-            xnf_data = json.loads(response.text)
+            nf_data = json.loads(response.text)
     except Exception as e:
         logger.debug(e)
-    return xnf_data
+    return nf_data
 
 
 def _get_aai_service_url():
@@ -91,52 +93,37 @@ def _get_aai_service_url():
         KeyError: if AAI env vars not found.
     """
     try:
-        aai_service = os.environ['AAI_SERVICE_HOST']
-        aai_ssl_port = os.environ['AAI_SERVICE_PORT_AAI_SSL']
+        aai_service = environ['AAI_SERVICE_HOST']
+        aai_ssl_port = environ['AAI_SERVICE_PORT_AAI_SSL']
         return f'https://{aai_service}:{aai_ssl_port}'
     except KeyError as e:
         logger.debug(f'Failed to get AAI env vars: {e}')
         raise
 
 
-def _filter_xnf_data(xnf_data, xnf_filter):
+def _filter_nf_data(nf_data, nf_filter):
     """
-    Returns a list of filtered xnfs using the xnf_filter .
+    Returns a list of filtered NetworkFunctions using the nf_filter.
 
     Args:
-        xnf_data: the xnf json data from AAI.
-        xnf_filter: the `XnfFilter <XnfFilter>` to be applied.
+        nf_data : the nf json data from AAI.
+        nf_filter: the `NetworkFunctionFilter <NetworkFunctionFilter>` to be applied.
 
     Returns:
-        set: a set of filtered xnfs.
+        set: a set of filtered NetworkFunctions.
 
     Raises:
         KeyError: if AAI data cannot be parsed.
     """
-    xnf_set = set()
+    nf_set = set()
     try:
-        for xnf in xnf_data['results']:
-            name_identifier = 'pnf-name' if xnf['node-type'] == 'pnf' else 'vnf-name'
-            if xnf_filter.is_xnf_in_filter(xnf['properties'].get(name_identifier)):
-                xnf_set.add(Xnf(xnf_name=xnf['properties'].get('name_identifier'),
-                                orchestration_status=xnf['properties'].get('orchestration-status')))
+        for nf in nf_data['results']:
+            name_identifier = 'pnf-name' if nf['node-type'] == 'pnf' else 'vnf-name'
+            if nf_filter.is_nf_in_filter(nf['properties'].get(name_identifier)):
+                nf_set.add(NetworkFunction(
+                    nf_name=nf['properties'].get(name_identifier),
+                    orchestration_status=nf['properties'].get('orchestration-status')))
     except KeyError as e:
         logger.debug(f'Failed to parse AAI data: {e}')
         raise
-    return xnf_set
-
-
-class Xnf:
-    def __init__(self, **kwargs):
-        """
-        Object representation of the XNF.
-        """
-        self.xnf_name = kwargs.get('xnf_name')
-        self.orchestration_status = kwargs.get('orchestration_status')
-
-    @classmethod
-    def xnf_def(cls):
-        return cls(xnf_name=None, orchestration_status=None)
-
-    def __str__(self):
-        return f'xnf-name: {self.xnf_name}, orchestration-status: {self.orchestration_status}'
+    return nf_set
diff --git a/components/pm-subscription-handler/pmsh_service/mod/config_handler.py b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py
index e9edbca4..1ce4b701 100755
--- a/components/pm-subscription-handler/pmsh_service/mod/config_handler.py
+++ b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py
@@ -22,7 +22,7 @@ from os import environ
 import requests
 from tenacity import retry, wait_fixed, stop_after_attempt
 
-from pmsh_service.mod import pmsh_logging as logger
+import mod.pmsh_logging as logger
 
 
 class ConfigHandler:
diff --git a/components/pm-subscription-handler/pmsh_service/mod/db_models.py b/components/pm-subscription-handler/pmsh_service/mod/db_models.py
new file mode 100755
index 00000000..479d40e5
--- /dev/null
+++ b/components/pm-subscription-handler/pmsh_service/mod/db_models.py
@@ -0,0 +1,88 @@
+# ============LICENSE_START===================================================
+#  Copyright (C) 2019-2020 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=====================================================
+
+from sqlalchemy import Column, Integer, String, ForeignKey
+from sqlalchemy.orm import relationship
+
+from mod import db
+
+
+class SubscriptionModel(db.Model):
+    __tablename__ = 'subscriptions'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    subscription_name = Column(String(100), unique=True)
+    status = Column(String(20))
+
+    nfs = relationship(
+        'NfSubRelationalModel',
+        cascade='all, delete-orphan',
+        backref='subscription')
+
+    def __init__(self, subscription_name, status):
+        self.subscription_name = subscription_name
+        self.status = status
+
+    def __repr__(self):
+        return f'Subscription: {self.subscription_name}  {self.status}'
+
+    def __eq__(self, other):
+        if isinstance(self, other.__class__):
+            return self.subscription_name == other.subscription_name
+        return False
+
+
+class NetworkFunctionModel(db.Model):
+    __tablename__ = 'network_functions'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    nf_name = Column(String(100), unique=True)
+    orchestration_status = Column(String(100))
+
+    subscriptions = relationship(
+        'NfSubRelationalModel',
+        cascade='all, delete-orphan',
+        backref='nf')
+
+    def __init__(self, nf_name, orchestration_status):
+        self.nf_name = nf_name
+        self.orchestration_status = orchestration_status
+
+    def __repr__(self):
+        return f'NetworkFunctionModel: {self.nf_name}, {self.orchestration_status}'
+
+
+class NfSubRelationalModel(db.Model):
+    __tablename__ = 'nf_to_sub_rel'
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    subscription_name = Column(
+        String,
+        ForeignKey(SubscriptionModel.subscription_name, ondelete='cascade', onupdate='cascade')
+    )
+    nf_name = Column(
+        String,
+        ForeignKey(NetworkFunctionModel.nf_name, ondelete='cascade', onupdate='cascade')
+    )
+    nf_sub_status = Column(String(20))
+
+    def __init__(self, subscription_name, nf_name, nf_sub_status=None):
+        self.subscription_name = subscription_name
+        self.nf_name = nf_name
+        self.nf_sub_status = nf_sub_status
+
+    def __repr__(self):
+        return f'NetworkFunctionSubscriptions: {self.subscription_name}, ' \
+            f'{self.nf_name}, {self.nf_sub_status}'
diff --git a/components/pm-subscription-handler/pmsh_service/mod/network_function.py b/components/pm-subscription-handler/pmsh_service/mod/network_function.py
new file mode 100755
index 00000000..64f614af
--- /dev/null
+++ b/components/pm-subscription-handler/pmsh_service/mod/network_function.py
@@ -0,0 +1,73 @@
+# ============LICENSE_START===================================================
+#  Copyright (C) 2020 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=====================================================
+
+from mod import pmsh_logging as logger, db
+from mod.db_models import NetworkFunctionModel
+
+
+class NetworkFunction:
+    def __init__(self, **kwargs):
+        """
+        Object representation of the NetworkFunction.
+        """
+        self.nf_name = kwargs.get('nf_name')
+        self.orchestration_status = kwargs.get('orchestration_status')
+
+    @classmethod
+    def nf_def(cls):
+        return cls(nf_name=None, orchestration_status=None)
+
+    def __str__(self):
+        return f'nf-name: {self.nf_name}, orchestration-status: {self.orchestration_status}'
+
+    def create(self):
+        """ Creates a NetworkFunction database entry
+        """
+        existing_nf = NetworkFunctionModel.query.filter(
+            NetworkFunctionModel.nf_name == self.nf_name).one_or_none()
+
+        if existing_nf is None:
+            new_nf = NetworkFunctionModel(nf_name=self.nf_name,
+                                          orchestration_status=self.orchestration_status)
+            db.session.add(new_nf)
+            db.session.commit()
+
+            return new_nf
+        else:
+            logger.debug(f'Network function {existing_nf} already exists,'
+                         f' returning this network function..')
+            return existing_nf
+
+    @staticmethod
+    def get(nf_name):
+        """ Retrieves a network function
+        Args:
+            nf_name (str): The network function name
+        Returns:
+            NetworkFunctionModel object else None
+        """
+        return NetworkFunctionModel.query.filter(
+            NetworkFunctionModel.nf_name == nf_name).one_or_none()
+
+    @staticmethod
+    def get_all():
+        """ Retrieves all network functions
+        Returns:
+            list: NetworkFunctionModel objects else empty
+        """
+        return NetworkFunctionModel.query.all()
diff --git a/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py b/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py
index f88ea137..f2d11d49 100644
--- a/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py
+++ b/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py
@@ -64,7 +64,7 @@ def get_module_logger(mod_name):
     return logger
 
 
-def create_loggers(logs_path="/var/log/ONAP/pmsh/logs"):
+def create_loggers(logs_path=''):
     """
     Public method to set the global logger, launched from Run
     This is *not* launched during unit testing, so unit tests do not
diff --git a/components/pm-subscription-handler/pmsh_service/mod/subscription.py b/components/pm-subscription-handler/pmsh_service/mod/subscription.py
index aa3318ac..265d90b8 100755
--- a/components/pm-subscription-handler/pmsh_service/mod/subscription.py
+++ b/components/pm-subscription-handler/pmsh_service/mod/subscription.py
@@ -1,5 +1,5 @@
 # ============LICENSE_START===================================================
-#  Copyright (C) 2019 Nordix Foundation.
+#  Copyright (C) 2019-2020 Nordix Foundation.
 # ============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,10 @@
 # ============LICENSE_END=====================================================
 import re
 
+import mod.pmsh_logging as logger
+from mod import db
+from mod.db_models import SubscriptionModel, NfSubRelationalModel
+
 
 class Subscription:
     def __init__(self, **kwargs):
@@ -41,21 +45,100 @@ class Subscription:
         clean_sub.update({'nfName': xnf_name, 'policyName': f'OP-{self.subscriptionName}'})
         return clean_sub
 
+    def create(self):
+        """ Creates a subscription database entry
+
+        Returns:
+            Subscription object
+        """
+        existing_subscription = (SubscriptionModel.query.filter(
+            SubscriptionModel.subscription_name == self.subscriptionName).one_or_none())
+
+        if existing_subscription is None:
+            new_subscription = SubscriptionModel(subscription_name=self.subscriptionName,
+                                                 status=self.administrativeState)
+
+            db.session.add(new_subscription)
+            db.session.commit()
+
+            return new_subscription
+
+        else:
+            logger.debug(f'Subscription {self.subscriptionName} already exists,'
+                         f' returning this subscription..')
+            return existing_subscription
+
+    def add_network_functions_to_subscription(self, nf_list):
+        """ Associates network functions to a Subscription
 
-class XnfFilter:
+        Args:
+            nf_list : A list of NetworkFunction objects.
+        """
+        current_sub = self.create()
+        logger.debug(f'Adding network functions to subscription {current_sub.subscription_name}')
+
+        for nf in nf_list:
+            current_nf = nf.create()
+
+            existing_entry = NfSubRelationalModel.query.filter(
+                NfSubRelationalModel.subscription_name == current_sub.subscription_name,
+                NfSubRelationalModel.nf_name == current_nf.nf_name).one_or_none()
+            if existing_entry is None:
+                new_nf_sub = NfSubRelationalModel(current_sub.subscription_name, nf.nf_name)
+                new_nf_sub.nf = current_nf
+                logger.debug(current_nf)
+                current_sub.nfs.append(new_nf_sub)
+
+        db.session.add(current_sub)
+        db.session.commit()
+
+    @staticmethod
+    def get(subscription_name):
+        """ Retrieves a subscription
+
+        Args:
+            subscription_name (str): The subscription name
+
+        Returns:
+            Subscription object else None
+        """
+        return SubscriptionModel.query.filter(
+            SubscriptionModel.subscription_name == subscription_name).one_or_none()
+
+    @staticmethod
+    def get_all():
+        """ Retrieves a list of subscriptions
+
+        Returns:
+            list: Subscription list else empty
+        """
+        return SubscriptionModel.query.all()
+
+    @staticmethod
+    def get_all_nfs_subscription_relations():
+        """ Retrieves all network function to subscription relations
+
+        Returns:
+            list: NetworkFunctions per Subscription list else empty
+        """
+        nf_per_subscriptions = NfSubRelationalModel.query.all()
+
+        return nf_per_subscriptions
+
+
+class NetworkFunctionFilter:
     def __init__(self, **kwargs):
         self.nf_sw_version = kwargs.get('swVersions')
         self.nf_names = kwargs.get('nfNames')
         self.regex_matcher = re.compile('|'.join(raw_regex for raw_regex in self.nf_names))
 
-    def is_xnf_in_filter(self, xnf_name):
-        """Match the xnf name against regex values in Subscription.nfFilter.nfNames
+    def is_nf_in_filter(self, nf_name):
+        """Match the nf name against regex values in Subscription.nfFilter.nfNames
 
         Args:
-            xnf_name: the AAI xnf name.
+            nf_name: the AAI nf name.
 
         Returns:
             bool: True if matched, else False.
         """
-
-        return self.regex_matcher.search(xnf_name)
+        return self.regex_matcher.search(nf_name)
diff --git a/components/pm-subscription-handler/pmsh_service/pmsh_service.py b/components/pm-subscription-handler/pmsh_service/pmsh_service.py
index d8a593fb..99689d01 100755
--- a/components/pm-subscription-handler/pmsh_service/pmsh_service.py
+++ b/components/pm-subscription-handler/pmsh_service/pmsh_service.py
@@ -15,17 +15,34 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 # ============LICENSE_END=====================================================
+import sys
 import time
 
+import mod.aai_client as aai_client
 import mod.pmsh_logging as logger
+from mod import db, create_prod_app
+from mod.config_handler import ConfigHandler
+from mod.subscription import Subscription
 
 
 def main():
-    logger.create_loggers()
+
+    try:
+        app = create_prod_app()
+        app.app_context().push()
+        db.create_all(app=app)
+
+        config_handler = ConfigHandler()
+        cbs_data = config_handler.get_config()
+        subscription, xnfs = aai_client.get_pmsh_subscription_data(cbs_data)
+        subscription.add_network_functions_to_subscription(xnfs)
+    except Exception as e:
+        logger.debug(f'Failed to Init PMSH: {e}')
+        sys.exit(e)
 
     while True:
-        time.sleep(30)
-        logger.debug("He's not the messiah, he's a very naughty boy!")
+        logger.debug(Subscription.get_all_nfs_subscription_relations())
+        time.sleep(5)
 
 
 if __name__ == '__main__':