From 8b3fc62050a344fe9a9c8909e4c672cb9aa3281d Mon Sep 17 00:00:00 2001 From: efiacor Date: Fri, 24 Jan 2020 13:19:01 +0000 Subject: Adding DB Init and setup Signed-off-by: efiacor Change-Id: Ie32efcf007c9b6fa25b0072019f4a91a841d9d0c Issue-ID: DCAEGEN2-1828 --- .../pmsh_service/mod/__init__.py | 48 ++++++++++- .../pmsh_service/mod/aai_client.py | 71 +++++++--------- .../pmsh_service/mod/config_handler.py | 2 +- .../pmsh_service/mod/db_models.py | 88 ++++++++++++++++++++ .../pmsh_service/mod/network_function.py | 73 ++++++++++++++++ .../pmsh_service/mod/pmsh_logging.py | 2 +- .../pmsh_service/mod/subscription.py | 97 ++++++++++++++++++++-- .../pmsh_service/pmsh_service.py | 23 ++++- 8 files changed, 347 insertions(+), 57 deletions(-) create mode 100755 components/pm-subscription-handler/pmsh_service/mod/db_models.py create mode 100755 components/pm-subscription-handler/pmsh_service/mod/network_function.py (limited to 'components/pm-subscription-handler/pmsh_service') diff --git a/components/pm-subscription-handler/pmsh_service/mod/__init__.py b/components/pm-subscription-handler/pmsh_service/mod/__init__.py index d2f6f2fd..f8b1c598 100644 --- a/components/pm-subscription-handler/pmsh_service/mod/__init__.py +++ b/components/pm-subscription-handler/pmsh_service/mod/__init__.py @@ -1,5 +1,5 @@ # ============LICENSE_START=================================================== -# Copyright (C) 2019 Nordix Foundation. +# Copyright (C) 2019-2020 Nordix Foundation. # ============================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,49 @@ # # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END===================================================== +import os +from urllib.parse import quote +from connexion import App +from flask_sqlalchemy import SQLAlchemy -# empty __init__.py so that pytest can add correct path to coverage report, -# -- per pytest best practice guideline +import mod.pmsh_logging as logger + +db = SQLAlchemy() +basedir = os.path.abspath(os.path.dirname(__file__)) + + +def create_prod_app(): + logger.create_loggers(os.getenv('LOGS_PATH')) + connex_app = App(__name__, specification_dir=basedir) + app = connex_app.app + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + app.config['SQLALCHEMY_RECORD_QUERIES'] = True + app.config['SQLALCHEMY_DATABASE_URI'] = get_db_connection_url() + db.init_app(app) + return app + + +def create_test_app(): + logger.create_loggers('./unit_test_logs') + connex_app = App(__name__, specification_dir=basedir) + app = connex_app.app + app.config['TESTING'] = True + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + app.config['SQLALCHEMY_RECORD_QUERIES'] = True + app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('TEST_DB_URL', 'sqlite://') + db.init_app(app) + return app + + +def get_db_connection_url(): + pg_host = os.getenv('PMSH_PG_URL') + pg_user = os.getenv('PMSH_PG_USERNAME') + pg_user_pass = os.getenv('PMSH_PG_PASSWORD') + pmsh_db_name = os.getenv('PMSH_DB_NAME', 'pmsh') + pmsh_db_port = os.getenv('PMSH_PG_PORT', '5432') + db_url = f'postgres+psycopg2://{quote(str(pg_user), safe="")}:' \ + f'{quote(str(pg_user_pass), safe="")}@{pg_host}:{pmsh_db_port}/{pmsh_db_name}' + if 'None' in db_url: + raise Exception(f'Invalid DB connection URL: {db_url} .. exiting app!') + return db_url diff --git a/components/pm-subscription-handler/pmsh_service/mod/aai_client.py b/components/pm-subscription-handler/pmsh_service/mod/aai_client.py index 8b51a712..747846f1 100755 --- a/components/pm-subscription-handler/pmsh_service/mod/aai_client.py +++ b/components/pm-subscription-handler/pmsh_service/mod/aai_client.py @@ -16,14 +16,15 @@ # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END===================================================== import json -import os import uuid +from os import environ import requests from requests.auth import HTTPBasicAuth import mod.pmsh_logging as logger -from mod.subscription import Subscription, XnfFilter +from mod.network_function import NetworkFunction +from mod.subscription import Subscription, NetworkFunctionFilter def get_pmsh_subscription_data(cbs_data): @@ -34,28 +35,29 @@ def get_pmsh_subscription_data(cbs_data): cbs_data: json app config from the Config Binding Service. Returns: - Subscription, set(Xnf): `Subscription` object, set of XNFs to be added. + Subscription, set(NetworkFunctions): `Subscription` object, + set of NetworkFunctions to be added. Raises: RuntimeError: if AAI data cannot be retrieved. """ - aai_xnf_data = _get_all_aai_xnf_data() - if aai_xnf_data: + aai_nf_data = _get_all_aai_nf_data() + if aai_nf_data: sub = Subscription(**cbs_data['policy']['subscription']) - xnfs = _filter_xnf_data(aai_xnf_data, XnfFilter(**sub.nfFilter)) + nfs = _filter_nf_data(aai_nf_data, NetworkFunctionFilter(**sub.nfFilter)) else: raise RuntimeError('Failed to get data from AAI') - return sub, xnfs + return sub, nfs -def _get_all_aai_xnf_data(): +def _get_all_aai_nf_data(): """ - Return queried xnf data from the AAI service. + Return queried nf data from the AAI service. Returns: json: the json response from AAI query, else None. """ - xnf_data = None + nf_data = None try: session = requests.Session() aai_endpoint = f'{_get_aai_service_url()}{"/aai/v16/query"}' @@ -74,10 +76,10 @@ def _get_all_aai_xnf_data(): data=json_data, params=params, verify=False) response.raise_for_status() if response.ok: - xnf_data = json.loads(response.text) + nf_data = json.loads(response.text) except Exception as e: logger.debug(e) - return xnf_data + return nf_data def _get_aai_service_url(): @@ -91,52 +93,37 @@ def _get_aai_service_url(): KeyError: if AAI env vars not found. """ try: - aai_service = os.environ['AAI_SERVICE_HOST'] - aai_ssl_port = os.environ['AAI_SERVICE_PORT_AAI_SSL'] + aai_service = environ['AAI_SERVICE_HOST'] + aai_ssl_port = environ['AAI_SERVICE_PORT_AAI_SSL'] return f'https://{aai_service}:{aai_ssl_port}' except KeyError as e: logger.debug(f'Failed to get AAI env vars: {e}') raise -def _filter_xnf_data(xnf_data, xnf_filter): +def _filter_nf_data(nf_data, nf_filter): """ - Returns a list of filtered xnfs using the xnf_filter . + Returns a list of filtered NetworkFunctions using the nf_filter. Args: - xnf_data: the xnf json data from AAI. - xnf_filter: the `XnfFilter ` to be applied. + nf_data : the nf json data from AAI. + nf_filter: the `NetworkFunctionFilter ` to be applied. Returns: - set: a set of filtered xnfs. + set: a set of filtered NetworkFunctions. Raises: KeyError: if AAI data cannot be parsed. """ - xnf_set = set() + nf_set = set() try: - for xnf in xnf_data['results']: - name_identifier = 'pnf-name' if xnf['node-type'] == 'pnf' else 'vnf-name' - if xnf_filter.is_xnf_in_filter(xnf['properties'].get(name_identifier)): - xnf_set.add(Xnf(xnf_name=xnf['properties'].get('name_identifier'), - orchestration_status=xnf['properties'].get('orchestration-status'))) + for nf in nf_data['results']: + name_identifier = 'pnf-name' if nf['node-type'] == 'pnf' else 'vnf-name' + if nf_filter.is_nf_in_filter(nf['properties'].get(name_identifier)): + nf_set.add(NetworkFunction( + nf_name=nf['properties'].get(name_identifier), + orchestration_status=nf['properties'].get('orchestration-status'))) except KeyError as e: logger.debug(f'Failed to parse AAI data: {e}') raise - return xnf_set - - -class Xnf: - def __init__(self, **kwargs): - """ - Object representation of the XNF. - """ - self.xnf_name = kwargs.get('xnf_name') - self.orchestration_status = kwargs.get('orchestration_status') - - @classmethod - def xnf_def(cls): - return cls(xnf_name=None, orchestration_status=None) - - def __str__(self): - return f'xnf-name: {self.xnf_name}, orchestration-status: {self.orchestration_status}' + return nf_set diff --git a/components/pm-subscription-handler/pmsh_service/mod/config_handler.py b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py index e9edbca4..1ce4b701 100755 --- a/components/pm-subscription-handler/pmsh_service/mod/config_handler.py +++ b/components/pm-subscription-handler/pmsh_service/mod/config_handler.py @@ -22,7 +22,7 @@ from os import environ import requests from tenacity import retry, wait_fixed, stop_after_attempt -from pmsh_service.mod import pmsh_logging as logger +import mod.pmsh_logging as logger class ConfigHandler: diff --git a/components/pm-subscription-handler/pmsh_service/mod/db_models.py b/components/pm-subscription-handler/pmsh_service/mod/db_models.py new file mode 100755 index 00000000..479d40e5 --- /dev/null +++ b/components/pm-subscription-handler/pmsh_service/mod/db_models.py @@ -0,0 +1,88 @@ +# ============LICENSE_START=================================================== +# Copyright (C) 2019-2020 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END===================================================== + +from sqlalchemy import Column, Integer, String, ForeignKey +from sqlalchemy.orm import relationship + +from mod import db + + +class SubscriptionModel(db.Model): + __tablename__ = 'subscriptions' + id = Column(Integer, primary_key=True, autoincrement=True) + subscription_name = Column(String(100), unique=True) + status = Column(String(20)) + + nfs = relationship( + 'NfSubRelationalModel', + cascade='all, delete-orphan', + backref='subscription') + + def __init__(self, subscription_name, status): + self.subscription_name = subscription_name + self.status = status + + def __repr__(self): + return f'Subscription: {self.subscription_name} {self.status}' + + def __eq__(self, other): + if isinstance(self, other.__class__): + return self.subscription_name == other.subscription_name + return False + + +class NetworkFunctionModel(db.Model): + __tablename__ = 'network_functions' + id = Column(Integer, primary_key=True, autoincrement=True) + nf_name = Column(String(100), unique=True) + orchestration_status = Column(String(100)) + + subscriptions = relationship( + 'NfSubRelationalModel', + cascade='all, delete-orphan', + backref='nf') + + def __init__(self, nf_name, orchestration_status): + self.nf_name = nf_name + self.orchestration_status = orchestration_status + + def __repr__(self): + return f'NetworkFunctionModel: {self.nf_name}, {self.orchestration_status}' + + +class NfSubRelationalModel(db.Model): + __tablename__ = 'nf_to_sub_rel' + id = Column(Integer, primary_key=True, autoincrement=True) + subscription_name = Column( + String, + ForeignKey(SubscriptionModel.subscription_name, ondelete='cascade', onupdate='cascade') + ) + nf_name = Column( + String, + ForeignKey(NetworkFunctionModel.nf_name, ondelete='cascade', onupdate='cascade') + ) + nf_sub_status = Column(String(20)) + + def __init__(self, subscription_name, nf_name, nf_sub_status=None): + self.subscription_name = subscription_name + self.nf_name = nf_name + self.nf_sub_status = nf_sub_status + + def __repr__(self): + return f'NetworkFunctionSubscriptions: {self.subscription_name}, ' \ + f'{self.nf_name}, {self.nf_sub_status}' diff --git a/components/pm-subscription-handler/pmsh_service/mod/network_function.py b/components/pm-subscription-handler/pmsh_service/mod/network_function.py new file mode 100755 index 00000000..64f614af --- /dev/null +++ b/components/pm-subscription-handler/pmsh_service/mod/network_function.py @@ -0,0 +1,73 @@ +# ============LICENSE_START=================================================== +# Copyright (C) 2020 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END===================================================== + +from mod import pmsh_logging as logger, db +from mod.db_models import NetworkFunctionModel + + +class NetworkFunction: + def __init__(self, **kwargs): + """ + Object representation of the NetworkFunction. + """ + self.nf_name = kwargs.get('nf_name') + self.orchestration_status = kwargs.get('orchestration_status') + + @classmethod + def nf_def(cls): + return cls(nf_name=None, orchestration_status=None) + + def __str__(self): + return f'nf-name: {self.nf_name}, orchestration-status: {self.orchestration_status}' + + def create(self): + """ Creates a NetworkFunction database entry + """ + existing_nf = NetworkFunctionModel.query.filter( + NetworkFunctionModel.nf_name == self.nf_name).one_or_none() + + if existing_nf is None: + new_nf = NetworkFunctionModel(nf_name=self.nf_name, + orchestration_status=self.orchestration_status) + db.session.add(new_nf) + db.session.commit() + + return new_nf + else: + logger.debug(f'Network function {existing_nf} already exists,' + f' returning this network function..') + return existing_nf + + @staticmethod + def get(nf_name): + """ Retrieves a network function + Args: + nf_name (str): The network function name + Returns: + NetworkFunctionModel object else None + """ + return NetworkFunctionModel.query.filter( + NetworkFunctionModel.nf_name == nf_name).one_or_none() + + @staticmethod + def get_all(): + """ Retrieves all network functions + Returns: + list: NetworkFunctionModel objects else empty + """ + return NetworkFunctionModel.query.all() diff --git a/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py b/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py index f88ea137..f2d11d49 100644 --- a/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py +++ b/components/pm-subscription-handler/pmsh_service/mod/pmsh_logging.py @@ -64,7 +64,7 @@ def get_module_logger(mod_name): return logger -def create_loggers(logs_path="/var/log/ONAP/pmsh/logs"): +def create_loggers(logs_path=''): """ Public method to set the global logger, launched from Run This is *not* launched during unit testing, so unit tests do not diff --git a/components/pm-subscription-handler/pmsh_service/mod/subscription.py b/components/pm-subscription-handler/pmsh_service/mod/subscription.py index aa3318ac..265d90b8 100755 --- a/components/pm-subscription-handler/pmsh_service/mod/subscription.py +++ b/components/pm-subscription-handler/pmsh_service/mod/subscription.py @@ -1,5 +1,5 @@ # ============LICENSE_START=================================================== -# Copyright (C) 2019 Nordix Foundation. +# Copyright (C) 2019-2020 Nordix Foundation. # ============================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,10 @@ # ============LICENSE_END===================================================== import re +import mod.pmsh_logging as logger +from mod import db +from mod.db_models import SubscriptionModel, NfSubRelationalModel + class Subscription: def __init__(self, **kwargs): @@ -41,21 +45,100 @@ class Subscription: clean_sub.update({'nfName': xnf_name, 'policyName': f'OP-{self.subscriptionName}'}) return clean_sub + def create(self): + """ Creates a subscription database entry + + Returns: + Subscription object + """ + existing_subscription = (SubscriptionModel.query.filter( + SubscriptionModel.subscription_name == self.subscriptionName).one_or_none()) + + if existing_subscription is None: + new_subscription = SubscriptionModel(subscription_name=self.subscriptionName, + status=self.administrativeState) + + db.session.add(new_subscription) + db.session.commit() + + return new_subscription + + else: + logger.debug(f'Subscription {self.subscriptionName} already exists,' + f' returning this subscription..') + return existing_subscription + + def add_network_functions_to_subscription(self, nf_list): + """ Associates network functions to a Subscription -class XnfFilter: + Args: + nf_list : A list of NetworkFunction objects. + """ + current_sub = self.create() + logger.debug(f'Adding network functions to subscription {current_sub.subscription_name}') + + for nf in nf_list: + current_nf = nf.create() + + existing_entry = NfSubRelationalModel.query.filter( + NfSubRelationalModel.subscription_name == current_sub.subscription_name, + NfSubRelationalModel.nf_name == current_nf.nf_name).one_or_none() + if existing_entry is None: + new_nf_sub = NfSubRelationalModel(current_sub.subscription_name, nf.nf_name) + new_nf_sub.nf = current_nf + logger.debug(current_nf) + current_sub.nfs.append(new_nf_sub) + + db.session.add(current_sub) + db.session.commit() + + @staticmethod + def get(subscription_name): + """ Retrieves a subscription + + Args: + subscription_name (str): The subscription name + + Returns: + Subscription object else None + """ + return SubscriptionModel.query.filter( + SubscriptionModel.subscription_name == subscription_name).one_or_none() + + @staticmethod + def get_all(): + """ Retrieves a list of subscriptions + + Returns: + list: Subscription list else empty + """ + return SubscriptionModel.query.all() + + @staticmethod + def get_all_nfs_subscription_relations(): + """ Retrieves all network function to subscription relations + + Returns: + list: NetworkFunctions per Subscription list else empty + """ + nf_per_subscriptions = NfSubRelationalModel.query.all() + + return nf_per_subscriptions + + +class NetworkFunctionFilter: def __init__(self, **kwargs): self.nf_sw_version = kwargs.get('swVersions') self.nf_names = kwargs.get('nfNames') self.regex_matcher = re.compile('|'.join(raw_regex for raw_regex in self.nf_names)) - def is_xnf_in_filter(self, xnf_name): - """Match the xnf name against regex values in Subscription.nfFilter.nfNames + def is_nf_in_filter(self, nf_name): + """Match the nf name against regex values in Subscription.nfFilter.nfNames Args: - xnf_name: the AAI xnf name. + nf_name: the AAI nf name. Returns: bool: True if matched, else False. """ - - return self.regex_matcher.search(xnf_name) + return self.regex_matcher.search(nf_name) diff --git a/components/pm-subscription-handler/pmsh_service/pmsh_service.py b/components/pm-subscription-handler/pmsh_service/pmsh_service.py index d8a593fb..99689d01 100755 --- a/components/pm-subscription-handler/pmsh_service/pmsh_service.py +++ b/components/pm-subscription-handler/pmsh_service/pmsh_service.py @@ -15,17 +15,34 @@ # # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END===================================================== +import sys import time +import mod.aai_client as aai_client import mod.pmsh_logging as logger +from mod import db, create_prod_app +from mod.config_handler import ConfigHandler +from mod.subscription import Subscription def main(): - logger.create_loggers() + + try: + app = create_prod_app() + app.app_context().push() + db.create_all(app=app) + + config_handler = ConfigHandler() + cbs_data = config_handler.get_config() + subscription, xnfs = aai_client.get_pmsh_subscription_data(cbs_data) + subscription.add_network_functions_to_subscription(xnfs) + except Exception as e: + logger.debug(f'Failed to Init PMSH: {e}') + sys.exit(e) while True: - time.sleep(30) - logger.debug("He's not the messiah, he's a very naughty boy!") + logger.debug(Subscription.get_all_nfs_subscription_relations()) + time.sleep(5) if __name__ == '__main__': -- cgit 1.2.3-korg