Integrating prometheus with Policy components

With this change, all PF components can export prometheus metrics
by default. The prometheus metrics servlet will be running on
/metrics servletPath..
If the metrics shouldn't be exported, just add a flag
    "promethus": false
in the restServerParameters of default configuration file.
This brings up prometheus servlet on /metrics servletPath.

Also as part of standard servlet support, "servletUriPath" and
"servletClass" are 2 other fields added to RestServerParameters
which can be used to add standard servlets. This can be revisited later
if needed.
Basically, any servlet can be added by passing these fields, for
example, to add prometheus metrics servlet on /test/metrics,
add the below to restServerParameters:
    "servletUriPath": "/test/metrics",
    "servletClass": "io.prometheus.client.exporter.MetricsServlet"

In addition, we can later go to individual components and add more
metrics, say for e.g.,
deployedPoliciesCounter/undeployedPoliciesCounter etc on PAP,
executedEvents/failedEvents etc on PDP and so on.
This will look something like below, for e.g. in policy-pap component:
 io.prometheus.client.Counter counter = Counter.build()
  .name("policies_deployed_total")
  .help("Number of policies deployed.").register()
Whenever a policy is deployed, just call
 counter.inc()

Usage of code like above in the individual component will expose
such data as well as part of the exposed metrics.

Change-Id: Id667f27b15c012398421ba657b5324cc1d82cf1f
Issue-ID: POLICY-3524
Signed-off-by: a.sreekumar <ajith.sreekumar@bell.ca>
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Signed-off-by: a.sreekumar <ajith.sreekumar@bell.ca>

3 files changed, 156 insertions, 6 deletions

diff --git a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpClientTest.java b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpClientTest.java
index 7e03b1f6..c9908edb 100644
--- a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpClientTest.java
+++ b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpClientTest.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  * ONAP
  * ================================================================================
- * Copyright (C) 2017-2020 AT&T Intellectual Property. All rights reserved.
+ * Copyright (C) 2017-2021 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2018 Samsung Electronics Co., Ltd.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
+import io.prometheus.client.exporter.MetricsServlet;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -377,7 +378,12 @@ public class HttpClientTest {
     public void testHttpAuthClientProps() throws Exception {
         final Properties httpProperties = new Properties();
 
+        /* PAP and PDP services */
+
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES, "PAP,PDP");
+
+        /* PAP server service configuration */
+
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
                         + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, LOCALHOST);
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
@@ -394,8 +400,15 @@ public class HttpClientTest {
                         PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
                                         + PolicyEndPointProperties.PROPERTY_HTTP_FILTER_CLASSES_SUFFIX,
                         TestFilter.class.getName());
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PAP
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
                         + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
+                        + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_CLASS_SUFFIX, MetricsServlet.class.getName());
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PAP
+                                           + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX,
+                                   "/pap/test/random/metrics");
+
+        /* PDP server service configuration */
 
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, LOCALHOST);
@@ -409,10 +422,19 @@ public class HttpClientTest {
                         PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PDP
                                         + PolicyEndPointProperties.PROPERTY_HTTP_REST_CLASSES_SUFFIX,
                         RestMockHealthCheck.class.getName());
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PAP
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PDP
+                                           + PolicyEndPointProperties.PROPERTY_HTTP_SWAGGER_SUFFIX, "true");
+        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + DOT_PDP
+            + PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX, "true");
+
+        /* PDP and PAP client services */
 
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES, "PAP,PDP");
+
+        /* PAP client service configuration */
+
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PAP
                         + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, LOCALHOST);
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PAP
@@ -428,13 +450,13 @@ public class HttpClientTest {
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PAP
                         + PolicyEndPointProperties.PROPERTY_MANAGED_SUFFIX, "true");
 
+        /* PDP client service configuration */
+
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_HTTP_HOST_SUFFIX, LOCALHOST);
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_HTTP_PORT_SUFFIX, "7778");
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PDP
-                        + PolicyEndPointProperties.PROPERTY_HTTP_URL_SUFFIX, "pdp");
-        httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_HTTP_HTTPS_SUFFIX, FALSE_STRING);
         httpProperties.setProperty(PolicyEndPointProperties.PROPERTY_HTTP_CLIENT_SERVICES + DOT_PDP
                         + PolicyEndPointProperties.PROPERTY_HTTP_AUTH_USERNAME_SUFFIX, "testpdp");
@@ -460,15 +482,28 @@ public class HttpClientTest {
         assertEquals(200, response.getStatus());
 
         final HttpClient clientPdp = HttpClientFactoryInstance.getClientFactory().get("PDP");
-        response = clientPdp.get("test");
+
+        response = clientPdp.get("pdp/test");
         assertEquals(500, response.getStatus());
 
+        response = clientPdp.get("metrics");
+        assertEquals(200, response.getStatus());
+
+        response = clientPdp.get("swagger.json");
+        assertEquals(200, response.getStatus());
+
         assertFalse(MyGsonProvider.hasWrittenSome());
 
         // try with empty path
         response = clientPap.get("");
         assertEquals(200, response.getStatus());
 
+        response = clientPap.get("random/metrics");
+        assertEquals(200, response.getStatus());
+
+        response = clientPap.get("metrics");
+        assertEquals(404, response.getStatus());
+
         // try it asynchronously, too
         MyCallback callback = new MyCallback();
         response = clientPap.get(callback, null).get();
diff --git a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpServerTest.java b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpServerTest.java
index 68c9dc8a..2d4de82b 100644
--- a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpServerTest.java
+++ b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/HttpServerTest.java
@@ -4,6 +4,7 @@
  * ================================================================================
  * Copyright (C) 2017-2020 AT&T Intellectual Property. All rights reserved.
  * Modifications Copyright (C) 2020 Nordix Foundation.
+ * Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,6 +32,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import com.google.gson.Gson;
+import io.prometheus.client.exporter.MetricsServlet;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
@@ -56,6 +58,9 @@ import org.slf4j.LoggerFactory;
  * HttpServletServer JUNIT tests.
  */
 public class HttpServerTest {
+    private static final String JVM_MEMORY_BYTES_USED = "jvm_memory_bytes_used";
+    private static final String METRICS_URI = "/metrics";
+    private static final String PROMETHEUS = "prometheus";
     private static final String LOCALHOST = "localhost";
     private static final String JSON_MEDIA = "application/json";
     private static final String YAML_MEDIA = YamlMessageBodyHandler.APPLICATION_YAML;
@@ -211,6 +216,79 @@ public class HttpServerTest {
         assertEquals(reqText, response);
     }
 
+    /**
+     * This test checks a server from a plain java servlet (note it uses prometheus as the sample server).
+     */
+    @Test
+    public void testStdServletServer() throws Exception {
+        logger.info("-- testStdServletServer() --");
+
+        HttpServletServer server = HttpServletServerFactoryInstance.getServerFactory()
+            .build(PROMETHEUS, LOCALHOST, port, "/", false, true);
+
+        server.addStdServletClass("/prom-generic-servlet/metrics", MetricsServlet.class.getName());
+        server.waitedStart(5000);
+
+        assertTrue(HttpServletServerFactoryInstance.getServerFactory().get(port).isAlive());
+        assertTrue(server.isPrometheus());
+
+        String response = http(portUrl + "/prom-generic-servlet/metrics");
+        assertThat(response).contains(JVM_MEMORY_BYTES_USED);
+    }
+
+    /**
+     * This test explicitly creates a prometheus server.
+     */
+    @Test
+    public void testExplicitPrometheusServer() throws Exception {
+        logger.info("-- testPrometheusServer() --");
+
+        HttpServletServer server = HttpServletServerFactoryInstance.getServerFactory()
+            .build(PROMETHEUS, LOCALHOST, port, "/", false, true);
+        server.setPrometheus(METRICS_URI);
+        server.waitedStart(5000);
+
+        assertTrue(HttpServletServerFactoryInstance.getServerFactory().get(port).isAlive());
+        assertTrue(server.isPrometheus());
+
+        String response = http(portUrl + METRICS_URI);
+        assertThat(response).contains(JVM_MEMORY_BYTES_USED);
+    }
+
+    /**
+     * This test is an all-in-one for a single server: prometheus, jax-rs, servlet, swagger, and filters.
+     */
+    @Test
+    public void testPrometheusJaxRsFilterSwaggerServer() throws Exception {
+        logger.info("-- testPrometheusServer() --");
+
+        HttpServletServer server = HttpServletServerFactoryInstance.getServerFactory()
+            .build(PROMETHEUS, LOCALHOST, port, "/", true, true);
+
+        server.addServletClass("/*", RestEchoService.class.getName());
+        server.addFilterClass("/*", TestFilter.class.getName());
+        server.setPrometheus(METRICS_URI);
+
+        server.waitedStart(5000);
+
+        assertTrue(HttpServletServerFactoryInstance.getServerFactory().get(port).isAlive());
+        assertTrue(server.isPrometheus());
+
+        String response = http(portUrl + METRICS_URI);
+        assertThat(response).contains(JVM_MEMORY_BYTES_USED);
+
+        RestEchoReqResp request = new RestEchoReqResp();
+        request.setRequestId(100);
+        request.setText(SOME_TEXT);
+        String reqText = gson.toJson(request);
+
+        response = http(portUrl + JUNIT_ECHO_FULL_REQUEST, JSON_MEDIA, reqText);
+        assertEquals(reqText, response);
+
+        response = http(portUrl + SWAGGER_JSON);
+        assertThat(response).contains("Swagger Server");
+    }
+
     @Test
     public void testJacksonClassServer() throws Exception {
         logger.info("-- testJacksonClassServer() --");
diff --git a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/RestServerTest.java b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/RestServerTest.java
index 16f17652..65af469b 100644
--- a/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/RestServerTest.java
+++ b/policy-endpoints/src/test/java/org/onap/policy/common/endpoints/http/server/test/RestServerTest.java
@@ -3,6 +3,7 @@
  * ONAP
  * ================================================================================
  * Copyright (C) 2019-2020 AT&T Intellectual Property. All rights reserved.
+ * Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,6 +32,7 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import io.prometheus.client.exporter.MetricsServlet;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.HttpURLConnection;
@@ -69,6 +71,7 @@ import org.onap.policy.common.utils.network.NetworkUtil;
 import org.powermock.reflect.Whitebox;
 
 public class RestServerTest {
+    private static final String METRICS_URI = "/metrics";
     private static final String SERVER1 = "my-server-A";
     private static final String SERVER2 = "my-server-B";
     private static final String FACTORY_FIELD = "factory";
@@ -231,6 +234,40 @@ public class RestServerTest {
         assertEquals(String.join(",", GsonMessageBodyHandler.class.getName(), YamlMessageBodyHandler.class.getName(),
                         JsonExceptionMapper.class.getName(), YamlExceptionMapper.class.getName()),
                         props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERIALIZATION_PROVIDER));
+        assertEquals("false", props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX));
+    }
+
+    @Test
+    public void testExplicitPrometheusAddedToProperty() {
+        when(params.isPrometheus()).thenReturn(true);
+        rest = new RestServer(params, Filter.class, Provider1.class, Provider2.class);
+        ArgumentCaptor<Properties> cap = ArgumentCaptor.forClass(Properties.class);
+        verify(serverFactory).build(cap.capture());
+
+        Properties props = cap.getValue();
+        String svcpfx = PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + PARAM_NAME;
+
+        assertEquals("true", props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX));
+        assertThat(props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX)).isBlank();
+        assertThat(props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_CLASS_SUFFIX)).isBlank();
+    }
+
+    @Test
+    public void testStandardSevletAddedToProperty() {
+        when(params.getServletUriPath()).thenReturn("/metrics");
+        when(params.getServletClass()).thenReturn(MetricsServlet.class.getName());
+        rest = new RestServer(params, Filter.class, Provider1.class, Provider2.class);
+        ArgumentCaptor<Properties> cap = ArgumentCaptor.forClass(Properties.class);
+        verify(serverFactory).build(cap.capture());
+
+        Properties props = cap.getValue();
+        String svcpfx = PolicyEndPointProperties.PROPERTY_HTTP_SERVER_SERVICES + "." + PARAM_NAME;
+
+        assertEquals("false", props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_PROMETHEUS_SUFFIX));
+        assertEquals(METRICS_URI,
+            props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_URIPATH_SUFFIX));
+        assertEquals(MetricsServlet.class.getName(),
+            props.getProperty(svcpfx + PolicyEndPointProperties.PROPERTY_HTTP_SERVLET_CLASS_SUFFIX));
     }
 
     @Test