[AAI] Export relevant key metrics for monitoring in Prometheus

Make the key metrics available to the monitoring system by
instrumenting the code.
The Key metrics are available via
/actuator/prometheus
/actuator/info
/actuator/health

Issue-ID: AAI-3343
Signed-off-by: Roger Yuan <roger.yuan@yoppworks.com>
Change-Id: I69f7eafb5105a04369526c70902ac7b676038c36

12 files changed, 160 insertions, 2 deletions

diff --git a/aai-resources/pom.xml b/aai-resources/pom.xml
index 0540c53..908e1ee 100644
--- a/aai-resources/pom.xml
+++ b/aai-resources/pom.xml
@@ -78,6 +78,10 @@
 
         <!-- End of Default ONAP Schema Properties -->
         <keycloak.version>11.0.2</keycloak.version>
+        <micrometer-spring-legacy.version>1.3.19</micrometer-spring-legacy.version>
+        <micrometer-core.version>1.6.6</micrometer-core.version>
+        <micrometer-registry-prometheus.version>1.6.6</micrometer-registry-prometheus.version>
+        <micrometer-jersey2>1.6.6</micrometer-jersey2>
         <!-- Setting some default value to not complain by editor but it will be overridden by gmaven plugin -->
     </properties>
     <profiles>
@@ -282,6 +286,34 @@
 
     <dependencies>
         <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-hateoas</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-spring-legacy</artifactId>
+            <version>${micrometer-spring-legacy.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-core</artifactId>
+            <version>${micrometer-core.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <version>${micrometer-registry-prometheus.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-jersey2</artifactId>
+            <version>${micrometer-jersey2}</version>
+        </dependency>
+        <dependency>
             <groupId>javax.jms</groupId>
             <artifactId>javax.jms-api</artifactId>
             <version>2.0.1</version>
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/BulkAddConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/BulkAddConsumer.java
index 0c316bf..e28cce9 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/BulkAddConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/BulkAddConsumer.java
@@ -21,9 +21,11 @@ package org.onap.aai.rest;
 
 import javax.ws.rs.Path;
 
+import io.micrometer.core.annotation.Timed;
 import org.onap.aai.restcore.HttpMethod;
 
 @Path("{version: v[1-9][0-9]*|latest}/bulkadd")
+@Timed
 public class BulkAddConsumer extends BulkConsumer {
 
 	@Override
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/BulkProcessConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/BulkProcessConsumer.java
index aac2deb..1a03656 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/BulkProcessConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/BulkProcessConsumer.java
@@ -21,9 +21,11 @@ package org.onap.aai.rest;
 
 import javax.ws.rs.Path;
 
+import io.micrometer.core.annotation.Timed;
 import org.onap.aai.restcore.HttpMethod;
 
 @Path("{version: v[1-9][0-9]*|latest}/bulkprocess")
+@Timed
 public class BulkProcessConsumer extends BulkConsumer {
 
 	@Override
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/ExampleConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/ExampleConsumer.java
index 74884e2..8541844 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/ExampleConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/ExampleConsumer.java
@@ -31,6 +31,7 @@ import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 import javax.ws.rs.core.UriInfo;
 
+import io.micrometer.core.annotation.Timed;
 import org.onap.aai.config.SpringContextAware;
 import org.onap.aai.exceptions.AAIException;
 import org.onap.aai.introspection.Introspector;
@@ -47,6 +48,7 @@ import org.onap.aai.restcore.RESTAPI;
  * The Class ExampleConsumer.
  */
 @Path("{version: v[1-9][0-9]*|latest}/examples")
+@Timed
 public class ExampleConsumer extends RESTAPI {
 
 	
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/LegacyMoxyConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/LegacyMoxyConsumer.java
index c9991cc..3881b09 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/LegacyMoxyConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/LegacyMoxyConsumer.java
@@ -19,6 +19,7 @@
  */
 package org.onap.aai.rest;
 
+import io.micrometer.core.annotation.Timed;
 import io.swagger.jaxrs.PATCH;
 import org.apache.commons.lang3.ObjectUtils;
 import org.javatuples.Pair;
@@ -58,6 +59,7 @@ import java.util.stream.Collectors;
  */
 @Controller
 @Path("{version: v[1-9][0-9]*|latest}")
+@Timed
 public class LegacyMoxyConsumer extends RESTAPI {
 
 	private static final Logger logger = LoggerFactory.getLogger(LegacyMoxyConsumer.class.getName());
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/URLFromVertexIdConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/URLFromVertexIdConsumer.java
index 9293569..acaec5e 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/URLFromVertexIdConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/URLFromVertexIdConsumer.java
@@ -19,6 +19,7 @@
  */
 package org.onap.aai.rest;
 
+import io.micrometer.core.annotation.Timed;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.onap.aai.config.SpringContextAware;
 import org.onap.aai.exceptions.AAIException;
@@ -45,6 +46,7 @@ import java.util.Iterator;
  * The Class URLFromVertexIdConsumer.
  */
 @Path("{version: v[1-9][0-9]*|latest}/generateurl")
+@Timed
 public class URLFromVertexIdConsumer extends RESTAPI {
 	private ModelType introspectorFactoryType = ModelType.MOXY;
 
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/VertexIdConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/VertexIdConsumer.java
index 6637d41..61853e3 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/VertexIdConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/VertexIdConsumer.java
@@ -19,6 +19,7 @@
  */
 package org.onap.aai.rest;
 
+import io.micrometer.core.annotation.Timed;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.javatuples.Pair;
 import org.onap.aai.config.SpringContextAware;
@@ -48,6 +49,7 @@ import java.util.List;
  * The Class VertexIdConsumer.
  */
 @Path("{version: v[1-9][0-9]*|latest}/resources")
+@Timed
 public class VertexIdConsumer extends RESTAPI {
 
 	private ModelType introspectorFactoryType = ModelType.MOXY;
diff --git a/aai-resources/src/main/java/org/onap/aai/rest/bulk/BulkSingleTransactionConsumer.java b/aai-resources/src/main/java/org/onap/aai/rest/bulk/BulkSingleTransactionConsumer.java
index b7b4873..a5e5d7c 100644
--- a/aai-resources/src/main/java/org/onap/aai/rest/bulk/BulkSingleTransactionConsumer.java
+++ b/aai-resources/src/main/java/org/onap/aai/rest/bulk/BulkSingleTransactionConsumer.java
@@ -23,6 +23,7 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonParser;
+import io.micrometer.core.annotation.Timed;
 import org.javatuples.Pair;
 import org.onap.aai.config.SpringContextAware;
 import org.onap.aai.exceptions.AAIException;
@@ -55,6 +56,7 @@ import java.net.URI;
 import java.util.*;
 
 @Path(value = "{version: v[1-9][0-9]*|latest}/bulk/single-transaction")
+@Timed
 public class BulkSingleTransactionConsumer extends RESTAPI {
 
 	private static final Set<String> validOperations = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("put", "patch", "delete")));
diff --git a/aai-resources/src/main/java/org/onap/aai/web/MicrometerConfiguration.java b/aai-resources/src/main/java/org/onap/aai/web/MicrometerConfiguration.java
new file mode 100644
index 0000000..0be4e0b
--- /dev/null
+++ b/aai-resources/src/main/java/org/onap/aai/web/MicrometerConfiguration.java
@@ -0,0 +1,60 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.web;
+import io.micrometer.core.instrument.Tag;
+import io.micrometer.core.instrument.Tags;
+import io.micrometer.jersey2.server.JerseyTags;
+import io.micrometer.jersey2.server.JerseyTagsProvider;
+import org.glassfish.jersey.server.ContainerResponse;
+import org.glassfish.jersey.server.monitoring.RequestEvent;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Configuration Class to add customized tags to http metrics scraped in /actuator/prometheus endpoint
+ */
+@Configuration
+public class MicrometerConfiguration {
+    private static final String TAG_AAI_URI = "aai_uri";
+    private static final String NOT_AVAILABLE = "NOT AVAILABLE";
+
+    @Bean
+    public JerseyTagsProvider jerseyTagsProvider() {
+        return new JerseyTagsProvider() {
+            @Override
+            public Iterable httpRequestTags(RequestEvent event) {
+                ContainerResponse response = event.getContainerResponse();
+                return Tags.of(JerseyTags.method(event.getContainerRequest()),
+                        JerseyTags.exception(event), JerseyTags.status(response), JerseyTags.outcome(response), getAaiUriTag(event));
+            }
+            private Tag getAaiUriTag(RequestEvent event) {
+                String aai_uri = event.getUriInfo().getRequestUri().toString();
+                if (aai_uri == null) {
+                    aai_uri = NOT_AVAILABLE;
+                }
+                return Tag.of(TAG_AAI_URI, aai_uri);
+            }
+            @Override
+            public Iterable<Tag> httpLongRequestTags(RequestEvent event) {
+                return Tags.of(JerseyTags.method(event.getContainerRequest()), JerseyTags.uri(event));
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/aai-resources/src/main/resources/application.properties b/aai-resources/src/main/resources/application.properties
index 7ed4c61..cd19be9 100644
--- a/aai-resources/src/main/resources/application.properties
+++ b/aai-resources/src/main/resources/application.properties
@@ -13,7 +13,8 @@ spring.autoconfigure.exclude=\
   org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration,\
   org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration,\
   org.keycloak.adapters.springboot.KeycloakAutoConfiguration,\
-  org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+  org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration,\
+  org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
 
 spring.jersey.application-path=${schema.uri.base.path}
 
@@ -91,6 +92,16 @@ schema.service.ssl.key-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1
 schema.service.ssl.trust-store-password=password(OBF:1vn21ugu1saj1v9i1v941sar1ugw1vo0)
 schema.service.versions.override=false
 
+#To Expose the Prometheus scraping endpoint
+management.server.port=8448
+#To Enable Actuator Endpoint, you can override this to True in OOM charts
+management.endpoints.enabled-by-default=false
+#To Enable Actuator Endpoint, you can override this in OOM Charts
+#management.endpoints.web.exposure.include=info, health, prometheus
+management.metrics.web.server.auto-time-requests=false
+management.metrics.tags.group_id=aai
+management.metrics.tags.app_id=${info.build.artifact}
+
 # Location of the cadi properties file should be specified here
 aaf.cadi.file=${server.local.startpath}/cadi.properties
 
diff --git a/aai-resources/src/test/java/org/onap/aai/rest/ConfigurationTest.java b/aai-resources/src/test/java/org/onap/aai/rest/ConfigurationTest.java
index ee33bfc..39f71fd 100644
--- a/aai-resources/src/test/java/org/onap/aai/rest/ConfigurationTest.java
+++ b/aai-resources/src/test/java/org/onap/aai/rest/ConfigurationTest.java
@@ -27,6 +27,7 @@ import org.onap.aai.ResourcesTestConfiguration;
 import org.onap.aai.restclient.PropertyPasswordConfiguration;
 import org.onap.aai.config.SpringContextAware;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.web.server.LocalServerPort;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.context.annotation.Import;
@@ -42,6 +43,7 @@ import java.util.Collections;
 import java.util.UUID;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Test REST requests against configuration resource
@@ -58,10 +60,14 @@ public class ConfigurationTest extends AbstractSpringRestTest {
     @LocalServerPort
     int randomPort;
 
+    @Value("${local.management.port}")
+    private int mgtPort;
+
     private HttpEntity<String> httpEntity;
     private HttpEntity<String> httpEntityPut;
     private HttpEntity<String> httpEntityPatch;
     private String baseUrl;
+    private String actuatorurl;
     private HttpHeaders headers;
     @Before
     public void setup() throws UnsupportedEncodingException {
@@ -79,6 +85,7 @@ public class ConfigurationTest extends AbstractSpringRestTest {
 
         httpEntity = new HttpEntity<String>(headers);
         baseUrl = "http://localhost:" + randomPort;
+        actuatorurl = "http://localhost:" + mgtPort;
     }
 
     @Test
@@ -157,4 +164,32 @@ public class ConfigurationTest extends AbstractSpringRestTest {
         
     }
     
+
+    @Test
+    public void TestManagementEndpointConfiguration() {
+        ResponseEntity responseEntity = null;
+        String responseBody = null;
+
+        //set Accept as text/plain in order to get access of endpoint "/actuator/prometheus"
+        headers.set("Accept", "text/plain");
+        httpEntity = new HttpEntity<String>(headers);
+        responseEntity = restTemplate.exchange(actuatorurl + "/actuator/prometheus", HttpMethod.GET, httpEntity, String.class);
+        responseBody = (String) responseEntity.getBody();
+        assertEquals(HttpStatus.OK, responseEntity.getStatusCode());
+        assertTrue(responseBody.contains("app_id"));
+        assertTrue(responseBody.contains("group_id"));
+
+        //Set Accept as MediaType.APPLICATION_JSON in order to get access of endpoint "/actuator/info" and "/actuator/health"
+        headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
+        httpEntity = new HttpEntity<String>(headers);
+        responseEntity = restTemplate.exchange(actuatorurl + "/actuator/info", HttpMethod.GET, httpEntity, String.class);
+        responseBody = (String) responseEntity.getBody();
+        assertEquals(HttpStatus.OK, responseEntity.getStatusCode());
+        assertTrue(responseBody.contains("aai-resources"));
+
+        responseEntity = restTemplate.exchange(actuatorurl + "/actuator/health", HttpMethod.GET, httpEntity, String.class);
+        responseBody = (String) responseEntity.getBody();
+        assertEquals(HttpStatus.OK, responseEntity.getStatusCode());
+        assertTrue(responseBody.contains("UP"));
+    }
 }
diff --git a/aai-resources/src/test/resources/application-test.properties b/aai-resources/src/test/resources/application-test.properties
index b96b9ce..8aee5e1 100644
--- a/aai-resources/src/test/resources/application-test.properties
+++ b/aai-resources/src/test/resources/application-test.properties
@@ -15,7 +15,8 @@ spring.autoconfigure.exclude=\
   org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration,\
   org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration,\
   org.keycloak.adapters.springboot.KeycloakAutoConfiguration,\
-  org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
+  org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration,\
+  org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration
 
 spring.profiles.active=production
 #The max number of active threads in this pool
@@ -72,3 +73,8 @@ schema.version.edge.label.start=v12
 schema.version.api.default=v15
 schema.translator.list=config
 
+#To expose the Prometheus scraping endpoint in unit test
+management.server.port=0
+management.endpoints.enabled-by-default=true
+management.endpoints.web.exposure.include=info, health, prometheus
+management.metrics.web.server.auto-time-requests=false