Prometheus metrics for OOF

Added OOF HPA metrics using prometheus client exporter
Added VNF statistics, HPA statistics with respect to flavor, vim etc.

Change-Id: I54030a148f97cb0c61f93bc552764c65236a467b
Issue-ID: OPTFRA-312
Signed-off-by: Dileep Ranganathan <dileep.ranganathan@intel.com>

8 files changed, 199 insertions, 5 deletions

diff --git a/conductor.conf b/conductor.conf
index b2f0078..bb43e55 100755
--- a/conductor.conf
+++ b/conductor.conf
@@ -422,3 +422,12 @@ server_url_version = v0
 # Extensions list to use (list value)
 extensions = multicloud
 
+
+[prometheus]
+
+#
+# From conductor
+#
+
+# Prometheus Metrics Endpoint (list value)
+#metrics_port = 8000,8001,8002,8003,8004
\ No newline at end of file
diff --git a/conductor/conductor/common/prometheus_metrics.py b/conductor/conductor/common/prometheus_metrics.py
new file mode 100644
index 0000000..6798cb1
--- /dev/null
+++ b/conductor/conductor/common/prometheus_metrics.py
@@ -0,0 +1,108 @@
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2018 Intel Corporation Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+''' Prometheus metrics '''
+from oslo_config import cfg
+from oslo_log import log
+from prometheus_client import Counter
+from prometheus_client import start_http_server
+
+LOG = log.getLogger(__name__)
+
+CONF = cfg.CONF
+
+METRICS_OPTS = [
+    cfg.ListOpt('metrics_port',
+               default=[8000, 8001, 8002, 8003, 8004],
+               help='Prometheus Metrics Endpoint')
+]
+
+CONF.register_opts(METRICS_OPTS, group='prometheus')
+
+MUSIC_VERSION = Counter('oof_music_version', 'Music Version', ['version'])
+
+VNF_COMPUTE_PROFILES = Counter(
+    'vnf_compute_profile',
+    'Compute Profiles used by VNFs over time',
+    ['customer_name', 'service_name', 'vnf_name', 'vnfc_name', 'flavor',
+     'cloud_region']
+)
+
+VNF_FAILURE = Counter(
+    'vnf_no_solution',
+    'No Homing solution',
+    ['customer_name', 'service_name']
+)
+
+VNF_SUB_OPTIMUM = Counter(
+    'vnf_sub_optimum_solution',
+    'VNFs with sub-optimum solution',
+    ['customer_name', 'service_name', 'vnf_name', 'vnfc_name']
+)
+
+VNF_SCORE = Counter(
+    'vnf_scores',
+    'HPA Scores of vnf',
+    ['customer_name', 'service_name', 'vnf_name', 'vnfc_name', 'hpa_score']
+)
+
+# HPA Matching stats
+# TODO (dileep)
+# Customer name is set as ONAP in R3.
+# General rule of thumb - if label not available. Label=N/A
+# Service name will be set as N/A for HPA metrics in R3.
+# vnf_name and vnfc_name will be N/A.
+# Currently this needs lots of changes. R4 will take care of this.
+HPA_FLAVOR_MATCH_SUCCESSFUL = Counter(
+    'flavor_match_successful',
+    'Number of times there is successful flavor match',
+    ['customer_name', 'service_name', 'vnf_name', 'vnfc_name', 'cloud_region',
+     'flavor']
+)
+
+HPA_FLAVOR_MATCH_UNSUCCESSFUL = Counter(
+    'flavor_match_unsuccessful',
+    'Number of times there is unsuccessful flavor match',
+    ['customer_name', 'service_name', 'vnf_name', 'vnfc_name', 'cloud_region',
+     'flavor']
+)
+
+HPA_CLOUD_REGION_SUCCESSFUL = Counter(
+    'cloud_region_successful',
+    'Number of times cloud region is selected successfully',
+    ['customer_name', 'service_name', 'cloud_region']
+)
+
+HPA_CLOUD_REGION_UNSUCCESSFUL = Counter(
+    'cloud_region_unsuccessful',
+    'Number of times no cloud region is selected',
+    ['customer_name', 'service_name', 'cloud_region']
+)
+
+
+def _init_metrics(port_index):
+    '''
+    Method to start Prometheus metrics endpoint http server
+    :param port_index: Used by splver, data, api, contorller
+    services to start metrics endpoint without conflicting
+    :return:
+    '''
+    start_http_server(int(CONF.prometheus.metrics_port[port_index]))
+    LOG.info("Prometheus metrics endpoint started at {}".format(
+        CONF.prometheus.metrics_port[port_index]))
diff --git a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py
index a77f0bf..26133bc 100644
--- a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py
+++ b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py
@@ -21,12 +21,12 @@
 '''Utility functions for
    Hardware Platform Awareness (HPA) constraint plugin'''
 
-# python imports
-import yaml
 import operator
 
-from conductor.i18n import _LE, _LI
-
+import conductor.common.prometheus_metrics as PC
+# python imports
+import yaml
+from conductor.i18n import _LI
 # Third-party library imports
 from oslo_log import log
 
@@ -55,6 +55,7 @@ class HpaMatchProvider(object):
     def __init__(self, candidate, req_cap_list):
         self.flavors_list = candidate['flavors']['flavor']
         self.req_cap_list = req_cap_list
+        self.m_vim_id = candidate.get('vim-id')
 
     # Find the flavor which has all the required capabilities
     def match_flavor(self):
@@ -76,6 +77,11 @@ class HpaMatchProvider(object):
                 flavor_cap_list = flavor['hpa-capabilities']
             except KeyError:
                 LOG.info(_LI("hpa-capabilities not found in flavor "))
+                # Metrics to Prometheus
+                m_flavor_name = flavor['flavor-name']
+                PC.HPA_FLAVOR_MATCH_UNSUCCESSFUL.labels('ONAP', 'N/A', 'N/A',
+                                                      'N/A', self.m_vim_id,
+                                                      m_flavor_name).inc()
                 continue
             for capability in CapabilityDataParser.get_item(flavor_cap_list,
                                                             'hpa-capability'):
@@ -88,6 +94,11 @@ class HpaMatchProvider(object):
                 if match_found:
                     LOG.info(_LI("Matching Flavor found '{}' for request - {}").
                              format(flavor['flavor-name'], self.req_cap_list))
+                    # Metrics to Prometheus
+                    m_flavor_name = flavor['flavor-name']
+                    PC.HPA_FLAVOR_MATCH_SUCCESSFUL.labels('ONAP', 'N/A', 'N/A',
+                                                          'N/A', self.m_vim_id,
+                                                          m_flavor_name).inc()
                     if score > max_score:
                         max_score = score
                         flavor_map = {"flavor-id": flavor['flavor-id'],
@@ -95,6 +106,20 @@ class HpaMatchProvider(object):
                                       "score": max_score}
                         directives = {"flavor_map": flavor_map,
                                       "directives": req_directives}
+                else:
+                    # Metrics to Prometheus
+                    m_flavor_name = flavor['flavor-name']
+                    PC.HPA_FLAVOR_MATCH_UNSUCCESSFUL.labels('ONAP', 'N/A',
+                                                            'N/A', 'N/A',
+                                                            self.m_vim_id,
+                                                            m_flavor_name).inc()
+            else:
+                # Metrics to Prometheus
+                m_flavor_name = flavor['flavor-name']
+                PC.HPA_FLAVOR_MATCH_UNSUCCESSFUL.labels('ONAP', 'N/A',
+                                                        'N/A', 'N/A',
+                                                        self.m_vim_id,
+                                                        m_flavor_name).inc()
         return directives
 
 
diff --git a/conductor/conductor/data/service.py b/conductor/conductor/data/service.py
index c0007fc..07fc873 100644
--- a/conductor/conductor/data/service.py
+++ b/conductor/conductor/data/service.py
@@ -20,6 +20,7 @@
 # import json
 # import os
 
+import conductor.common.prometheus_metrics as PC
 import cotyledon
 from conductor import messaging
 # from conductor import __file__ as conductor_root
@@ -71,6 +72,10 @@ class DataServiceLauncher(object):
         """Initializer."""
 
         self.conf = conf
+
+        # Initialize Prometheus metrics Endpoint
+        # Data service uses index 0
+        PC._init_metrics(0)
         self.init_extension_managers(conf)
 
 
@@ -489,11 +494,18 @@ class DataEndpoint(object):
                         "inventory provider: {} for candidate: {}").format(
                         self.ip_ext_manager.names()[0],
                         candidate_list[i].get("candidate_id")))
+
+            # Metrics to Prometheus
+            m_vim_id = candidate_list[i].get("vim-id")
             if not flavor_info:
                 discard_set.add(candidate_list[i].get("candidate_id"))
+                PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels('ONAP', 'N/A',
+                                                        m_vim_id).inc()
             else:
                 if not flavor_info.get("flavor-name"):
                     discard_set.add(candidate_list[i].get("candidate_id"))
+                    PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels('ONAP', 'N/A',
+                                                            m_vim_id).inc()
                 else:
                     if not candidate_list[i].get("flavor_map"):
                         candidate_list[i]["flavor_map"] = {}
@@ -510,6 +522,10 @@ class DataEndpoint(object):
                         candidate_list[i]["hpa_score"] = 0
                     candidate_list[i]["hpa_score"] += flavor_info.get("score")
 
+                    # Metrics to Prometheus
+                    PC.HPA_CLOUD_REGION_SUCCESSFUL.labels('ONAP', 'N/A',
+                                                          m_vim_id).inc()
+
         # return candidates not in discard set
         candidate_list[:] = [c for c in candidate_list
                              if c['candidate_id'] not in discard_set]
diff --git a/conductor/conductor/opts.py b/conductor/conductor/opts.py
index 52624cf..d34fbcc 100644
--- a/conductor/conductor/opts.py
+++ b/conductor/conductor/opts.py
@@ -22,6 +22,7 @@ import itertools
 import conductor.api.app
 import conductor.common.music.api
 import conductor.common.music.messaging.component
+import conductor.common.prometheus_metrics
 import conductor.common.sms
 import conductor.conf.inventory_provider
 import conductor.conf.service_controller
@@ -70,4 +71,5 @@ def list_opts():
         ('solver', conductor.solver.service.SOLVER_OPTS),
         ('reservation', conductor.reservation.service.reservation_OPTS),
         ('aaf_sms', conductor.common.sms.AAF_SMS_OPTS),
+        ('prometheus', conductor.common.prometheus_metrics.METRICS_OPTS),
     ]
diff --git a/conductor/conductor/solver/optimizer/constraints/hpa.py b/conductor/conductor/solver/optimizer/constraints/hpa.py
index 106953b..a6032b7 100644
--- a/conductor/conductor/solver/optimizer/constraints/hpa.py
+++ b/conductor/conductor/solver/optimizer/constraints/hpa.py
@@ -23,6 +23,7 @@
 
 # python imports
 
+import conductor.common.prometheus_metrics as PC
 from conductor.i18n import _LE, _LI
 # Conductor imports
 from conductor.solver.optimizer.constraints import constraint
@@ -67,6 +68,10 @@ class HPA(constraint.Constraint):
             if not response:
                 LOG.error(_LE("No matching candidates for HPA exists").format(
                     id))
+
+                # Metrics to Prometheus
+                PC.HPA_CLOUD_REGION_UNSUCCESSFUL.labels('ONAP', 'N/A',
+                                                        'ALL').inc()
                 break
                 # No need to continue.
 
diff --git a/conductor/conductor/solver/service.py b/conductor/conductor/solver/service.py
index 5fc9d29..bb63d5a 100644
--- a/conductor/conductor/solver/service.py
+++ b/conductor/conductor/solver/service.py
@@ -19,6 +19,7 @@
 
 import collections
 
+import conductor.common.prometheus_metrics as PC
 import cotyledon
 import json
 import time
@@ -151,6 +152,10 @@ class SolverServiceLauncher(object):
         #self.GroupRules = base.create_dynamic_model(
         #    keyspace=conf.keyspace, baseclass=group_rules.GroupRules, classname="GroupRules")
 
+        # Initialize Prometheus metrics Endpoint
+        # Solver service uses index 1
+        PC._init_metrics(1)
+
         if not self.Plan:
             raise
         if not self.OrderLock:
@@ -447,6 +452,11 @@ class SolverService(cotyledon.Service):
                 # Update the plan status
                 p.status = self.Plan.NOT_FOUND
                 p.message = message
+
+                # Metrics to Prometheus
+                m_svc_name = p.template['parameters'].get('service_name', 'N/A')
+                PC.VNF_FAILURE.labels('ONAP', m_svc_name).inc()
+
                 while 'FAILURE' in _is_success:
                     _is_success = p.update(condition=self.solver_owner_condition)
                     LOG.info(_LI("Plan serach failed, changing the template status from solving to not found, "
@@ -504,6 +514,24 @@ class SolverService(cotyledon.Service):
                                 rec["attributes"]["directives"] = \
                                     self.set_flavor_in_flavor_directives(
                                         resource.get("flavor_map"), resource.get("all_directives"))
+
+                                # Metrics to Prometheus
+                                m_vim_id = resource.get("vim-id")
+                                m_hpa_score = resource.get("hpa_score", 0)
+                                m_svc_name = p.template['parameters'].get(
+                                    'service_name', 'N/A')
+                                for vnfc, flavor in resource.get("flavor_map"):
+                                    PC.VNF_COMPUTE_PROFILES.labels('ONAP',
+                                                                   m_svc_name,
+                                                                   demand_name,
+                                                                   vnfc,
+                                                                   flavor,
+                                                                   m_vim_id).inc()
+
+                                PC.VNF_SCORE.labels('ONAP', m_svc_name,
+                                                    demand_name,
+                                                    m_hpa_score).inc()
+
                             if resource.get('conflict_id'):
                                 rec["candidate"]["conflict_id"] = resource.get("conflict_id")
 
diff --git a/conductor/requirements.txt b/conductor/requirements.txt
index 6bc9dba..52ed4ed 100644
--- a/conductor/requirements.txt
+++ b/conductor/requirements.txt
@@ -24,4 +24,5 @@ six>=1.9.0 # MIT, also required by futurist
 stevedore>=1.9.0 # Apache-2.0, also required by oslo.config
 WebOb>=1.2.3 # MIT
 onapsmsclient>=0.0.3
-Flask>=0.11.1
\ No newline at end of file
+Flask>=0.11.1
+prometheus-client>=0.3.1
\ No newline at end of file