Integrating prometheus with Policy components

With this change, all PF components can export prometheus metrics
by default. The prometheus metrics servlet will be running on
/metrics servletPath..
If the metrics shouldn't be exported, just add a flag
    "promethus": false
in the restServerParameters of default configuration file.
This brings up prometheus servlet on /metrics servletPath.

Also as part of standard servlet support, "servletUriPath" and
"servletClass" are 2 other fields added to RestServerParameters
which can be used to add standard servlets. This can be revisited later
if needed.
Basically, any servlet can be added by passing these fields, for
example, to add prometheus metrics servlet on /test/metrics,
add the below to restServerParameters:
    "servletUriPath": "/test/metrics",
    "servletClass": "io.prometheus.client.exporter.MetricsServlet"

In addition, we can later go to individual components and add more
metrics, say for e.g.,
deployedPoliciesCounter/undeployedPoliciesCounter etc on PAP,
executedEvents/failedEvents etc on PDP and so on.
This will look something like below, for e.g. in policy-pap component:
 io.prometheus.client.Counter counter = Counter.build()
  .name("policies_deployed_total")
  .help("Number of policies deployed.").register()
Whenever a policy is deployed, just call
 counter.inc()

Usage of code like above in the individual component will expose
such data as well as part of the exposed metrics.

Change-Id: Id667f27b15c012398421ba657b5324cc1d82cf1f
Issue-ID: POLICY-3524
Signed-off-by: a.sreekumar <ajith.sreekumar@bell.ca>
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Signed-off-by: a.sreekumar <ajith.sreekumar@bell.ca>

8 files changed, 114 insertions, 34 deletions

diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/HttpServletServer.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/HttpServletServer.java
index 49dfea06..23c2b54a 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/HttpServletServer.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/HttpServletServer.java
@@ -4,6 +4,7 @@
  * ================================================================================
  * Copyright (C) 2017-2019 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2020 Nordix Foundation.
+ * Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -90,6 +91,14 @@ public interface HttpServletServer extends Startable {
     void addServletClass(String servletPath, String restClass);
 
     /**
+     * Adds a Java Servlet.
+     *
+     * @param servletPath servlet path
+     * @param plainServletClass servlet class
+     */
+    void addStdServletClass(String servletPath, String plainServletClass);
+
+    /**
      * Adds a package containing JAX-RS classes to serve REST requests.
      *
      * @param servletPath servlet path
@@ -123,4 +132,14 @@ public interface HttpServletServer extends Startable {
      * @throws InterruptedException if the blocking operation is interrupted
      */
     boolean waitedStart(long maxWaitTime) throws InterruptedException;
+
+    /**
+     * Are prometheus metrics enabled?.
+     */
+    public boolean isPrometheus();
+
+    /**
+     * Enable prometheus metrics.
+     */
+    public void setPrometheus(String metricsPath);
 }
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/IndexedHttpServletServerFactory.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/IndexedHttpServletServerFactory.java
index e977f02c..86f5fb04 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/IndexedHttpServletServerFactory.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/IndexedHttpServletServerFactory.java
@@ -4,6 +4,7 @@
  * ================================================================================
  * Copyright (C) 2017-2019, 2021 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2020 Nordix Foundation.
+ * Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -137,9 +138,12 @@ class IndexedHttpServletServerFactory implements HttpServletServerFactory {
         final var restUriPath = props.getString(PolicyEndPointProperties.PROPERTY_HTTP_REST_URIPATH_SUFFIX, null);
 
         addFilterClasses(props, service, restUriPath);
-        addServletClasses(props, service, restUriPath);
+        addRestServletClasses(props, service, restUriPath);
         addServletPackages(props, service, restUriPath);
 
+        addServletClass(props, service);
+        setPrometheus(props, service);
+
         serviceList.add(service);
     }
 
@@ -167,6 +171,12 @@ class IndexedHttpServletServerFactory implements HttpServletServerFactory {
         }
     }
 
+    private void setPrometheus(PropertyUtils props, HttpServletServer service) {
+        if (props.getBoolean(PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX, false)) {
+            service.setPrometheus("/metrics");
+        }
+    }
+
     private void addFilterClasses(PropertyUtils props, HttpServletServer service, final String restUriPath) {
 
         final var filterClasses =
@@ -179,8 +189,7 @@ class IndexedHttpServletServerFactory implements HttpServletServerFactory {
         }
     }
 
-    private void addServletClasses(PropertyUtils props, HttpServletServer service, final String restUriPath) {
-
+    private void addRestServletClasses(PropertyUtils props, HttpServletServer service, final String restUriPath) {
         final var restClasses = props.getString(PolicyEndPointProperties.PROPERTY_HTTP_REST_CLASSES_SUFFIX, null);
 
         if (!StringUtils.isBlank(restClasses)) {
@@ -190,6 +199,15 @@ class IndexedHttpServletServerFactory implements HttpServletServerFactory {
         }
     }
 
+    private void addServletClass(PropertyUtils props, HttpServletServer service) {
+        var servletClass = props.getString(PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_CLASS_SUFFIX, null);
+        var servletUriPath = props.getString(PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX, null);
+
+        if (!StringUtils.isBlank(servletClass) && !StringUtils.isBlank(servletUriPath)) {
+            service.addStdServletClass(servletUriPath, servletClass);
+        }
+    }
+
     private void addServletPackages(PropertyUtils props, HttpServletServer service, final String restUriPath) {
 
         final var restPackages = props.getString(PolicyEndPointProperties.PROPERTY_HTTP_REST_PACKAGES_SUFFIX, null);
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/RestServer.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/RestServer.java
index 70d45112..cb50bb33 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/RestServer.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/RestServer.java
@@ -2,6 +2,7 @@
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2019 Nordix Foundation.
  *  Modifications Copyright (C) 2019-2021 AT&T Intellectual Property.
+ *  Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +24,7 @@ package org.onap.policy.common.endpoints.http.server;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.stream.Collectors;
 import lombok.ToString;
@@ -105,6 +107,13 @@ public class RestServer extends ServiceManagerContainer {
         props.setProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERIALIZATION_PROVIDER,
                         String.join(",", GsonMessageBodyHandler.class.getName(), YamlMessageBodyHandler.class.getName(),
                                         JsonExceptionMapper.class.getName(), YamlExceptionMapper.class.getName()));
+
+        props.setProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX,
+            Optional.ofNullable(restServerParameters.getServletUriPath()).orElse(""));
+        props.setProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_CLASS_SUFFIX,
+            Optional.ofNullable(restServerParameters.getServletClass()).orElse(""));
+        props.setProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX,
+            String.valueOf(restServerParameters.isPrometheus()));
         return props;
     }
 
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyJerseyServer.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyJerseyServer.java
index 467fd864..61aeadac 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyJerseyServer.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyJerseyServer.java
@@ -4,6 +4,7 @@
  * ================================================================================
  * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2019-2020 Nordix Foundation.
+ * Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,8 +23,6 @@
 package org.onap.policy.common.endpoints.http.server.internal;
 
 import io.swagger.jersey.config.JerseyJaxrsConfig;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jetty.servlet.ServletHolder;
 import org.glassfish.jersey.server.ServerProperties;
@@ -81,11 +80,6 @@ public class JettyJerseyServer extends JettyServletServer {
     protected static Logger logger = LoggerFactory.getLogger(JettyJerseyServer.class);
 
     /**
-     * Container for servlets.
-     */
-    protected final Map<String, ServletHolder> servlets = new HashMap<>();
-
-    /**
      * Swagger ID.
      */
     protected String swaggerId = null;
@@ -121,7 +115,7 @@ public class JettyJerseyServer extends JettyServletServer {
      */
     protected void attachSwaggerServlet(boolean https) {
 
-        ServletHolder swaggerServlet = context.addServlet(JerseyJaxrsConfig.class, "/");
+        ServletHolder swaggerServlet = getServlet(JerseyJaxrsConfig.class, "/");
 
         String hostname = this.connector.getHost();
         if (StringUtils.isBlank(hostname) || hostname.equals(NetworkUtil.IPV4_WILDCARD_ADDRESS)) {
@@ -149,15 +143,10 @@ public class JettyJerseyServer extends JettyServletServer {
      * @throws IllegalArgumentException if invalid arguments are provided
      */
     protected synchronized ServletHolder getServlet(String servletPath) {
-
-        return servlets.computeIfAbsent(servletPath, key -> {
-
-            ServletHolder jerseyServlet =
-                    context.addServlet(org.glassfish.jersey.servlet.ServletContainer.class, servletPath);
-            jerseyServlet.setInitOrder(0);
-
-            return jerseyServlet;
-        });
+        ServletHolder jerseyServlet =
+                super.getServlet(org.glassfish.jersey.servlet.ServletContainer.class, servletPath);
+        jerseyServlet.setInitOrder(0);
+        return jerseyServlet;
     }
 
     @Override
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyServletServer.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyServletServer.java
index e4b51372..c5af20cb 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyServletServer.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyServletServer.java
@@ -4,7 +4,7 @@
  * ================================================================================
  * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2019-2020 Nordix Foundation.
- * Modifications Copyright (C) 2020 Bell Canada. All rights reserved.
+ * Modifications Copyright (C) 2020-2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,9 +22,15 @@
 
 package org.onap.policy.common.endpoints.http.server.internal;
 
+import io.prometheus.client.exporter.MetricsServlet;
+import io.prometheus.client.hotspot.DefaultExports;
 import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
 import javax.servlet.DispatcherType;
+import javax.servlet.Servlet;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.ToString;
 import org.eclipse.jetty.security.ConstraintMapping;
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
@@ -40,6 +46,7 @@ import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.Slf4jRequestLogWriter;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
 import org.eclipse.jetty.util.security.Constraint;
 import org.eclipse.jetty.util.security.Credential;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
@@ -125,6 +132,11 @@ public abstract class JettyServletServer implements HttpServletServer, Runnable
     protected Thread jettyThread;
 
     /**
+     * Container for default servlets.
+     */
+    protected final Map<String, ServletHolder> servlets = new HashMap<>();
+
+    /**
      * Start condition.
      */
     @ToString.Exclude
@@ -209,6 +221,18 @@ public abstract class JettyServletServer implements HttpServletServer, Runnable
         context.addFilter(filterClass, tempFilterPath, EnumSet.of(DispatcherType.INCLUDE, DispatcherType.REQUEST));
     }
 
+    protected ServletHolder getServlet(@NonNull  Class<? extends Servlet> servlet, @NonNull  String servletPath) {
+        synchronized (servlets) {
+            return servlets.computeIfAbsent(servletPath, key -> context.addServlet(servlet, servletPath));
+        }
+    }
+
+    protected ServletHolder getServlet(String servletClass, String servletPath) {
+        synchronized (servlets) {
+            return servlets.computeIfAbsent(servletPath, key -> context.addServlet(servletClass, servletPath));
+        }
+    }
+
     /**
      * Returns the https connector.
      *
@@ -472,11 +496,32 @@ public abstract class JettyServletServer implements HttpServletServer, Runnable
     }
 
     @Override
-    public void addServletClass(String servletPath, String restClass) {
+    public void addServletClass(String servletPath, String servletClass) {
         throw new UnsupportedOperationException("addServletClass()" + NOT_SUPPORTED);
     }
 
     @Override
+    public void addStdServletClass(@NonNull String servletPath, @NonNull String plainServletClass) {
+        this.getServlet(plainServletClass, servletPath);
+    }
+
+    @Override
+    public void setPrometheus(String metricsPath) {
+        this.getServlet(MetricsServlet.class, metricsPath);
+        DefaultExports.initialize();
+    }
+
+    @Override
+    public boolean isPrometheus() {
+        for (ServletHolder servlet : context.getServletHandler().getServlets()) {
+            if (MetricsServlet.class.getName().equals(servlet.getClassName())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
     public void addServletPackage(String servletPath, String restPackage) {
         throw new UnsupportedOperationException("addServletPackage()" + NOT_SUPPORTED);
     }
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyStaticResourceServer.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyStaticResourceServer.java
index f0d6ba2c..c335247b 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyStaticResourceServer.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/http/server/internal/JettyStaticResourceServer.java
@@ -21,8 +21,6 @@
 
 package org.onap.policy.common.endpoints.http.server.internal;
 
-import java.util.HashMap;
-import java.util.Map;
 import lombok.ToString;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jetty.servlet.DefaultServlet;
@@ -57,11 +55,6 @@ public class JettyStaticResourceServer extends JettyServletServer {
     protected static Logger logger = LoggerFactory.getLogger(JettyStaticResourceServer.class);
 
     /**
-     * Container for default servlets.
-     */
-    protected final Map<String, ServletHolder> servlets = new HashMap<>();
-
-    /**
      * Constructor.
      *
      * @param name name
@@ -86,14 +79,13 @@ public class JettyStaticResourceServer extends JettyServletServer {
      * @throws IllegalArgumentException if invalid arguments are provided
      */
     protected synchronized ServletHolder getDefaultServlet(String servletPath) {
-
-        return servlets.computeIfAbsent(servletPath, key -> context.addServlet(DefaultServlet.class, servletPath));
+        return super.getServlet(DefaultServlet.class, servletPath);
     }
 
     @Override
-    public synchronized void addServletResource(String servletPath, String resoureBase) {
+    public synchronized void addServletResource(String servletPath, String resourceBase) {
 
-        if (StringUtils.isBlank(resoureBase)) {
+        if (StringUtils.isBlank(resourceBase)) {
             throw new IllegalArgumentException("No resourceBase provided");
         }
 
@@ -103,7 +95,7 @@ public class JettyStaticResourceServer extends JettyServletServer {
 
         ServletHolder defaultServlet = this.getDefaultServlet(servletPath);
 
-        defaultServlet.setInitParameter(SERVLET_HOLDER_RESOURCE_BASE, resoureBase);
+        defaultServlet.setInitParameter(SERVLET_HOLDER_RESOURCE_BASE, resourceBase);
         defaultServlet.setInitParameter(SERVLET_HOLDER_DIR_ALLOWED, "false");
         defaultServlet.setInitParameter(SERVLET_HOLDER_PATH_INFO_ONLY, "true");
 
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/parameters/RestServerParameters.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/parameters/RestServerParameters.java
index 57aeb9dc..671a1541 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/parameters/RestServerParameters.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/parameters/RestServerParameters.java
@@ -2,6 +2,7 @@
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2019 Nordix Foundation.
  *  Modifications Copyright (C) 2020-2021 AT&T Intellectual Property. All rights reserved.
+ *  Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -45,6 +46,9 @@ public class RestServerParameters extends ParameterGroupImpl {
     private String password;
     private boolean https;
     private boolean aaf;
+    private boolean prometheus = true;
+    private String servletClass;
+    private String servletUriPath;
 
     public RestServerParameters() {
         super(RestServerParameters.class.getSimpleName());
diff --git a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/properties/PolicyEndPointProperties.java b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/properties/PolicyEndPointProperties.java
index b373cfce..08ed2624 100644
--- a/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/properties/PolicyEndPointProperties.java
+++ b/policy-endpoints/src/main/java/org/onap/policy/common/endpoints/properties/PolicyEndPointProperties.java
@@ -96,6 +96,10 @@ public final class PolicyEndPointProperties {
     public static final String PROPERTY_HTTP_REST_PACKAGES_SUFFIX = ".restPackages";
     public static final String PROPERTY_HTTP_REST_URIPATH_SUFFIX = ".restUriPath";
 
+    public static final String PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX = ".servletUriPath";
+    public static final String PROPERTY_HTTP_SERVLET_CLASS_SUFFIX = ".servletClass";
+    public static final String PROPERTY_HTTP_PROMETHEUS_SUFFIX = ".prometheus";
+
     public static final String PROPERTY_HTTP_HTTPS_SUFFIX = ".https";
     public static final String PROPERTY_HTTP_SWAGGER_SUFFIX = ".swagger";