Added OpenTelemetry to CPS

Change-Id: I192fa53e293ea43cdff92ebd44d0382eb290bb76
Signed-off-by: david.mcweeney <david.mcweeney@est.tech>
Issue-ID: CPS-2172

9 files changed, 295 insertions, 5 deletions

diff --git a/cps-application/pom.xml b/cps-application/pom.xml
index 6804c7de65..abcb88f4a3 100644
--- a/cps-application/pom.xml
+++ b/cps-application/pom.xml
@@ -75,7 +75,7 @@
         </dependency>
         <dependency>
             <groupId>io.micrometer</groupId>
-            <artifactId>micrometer-tracing-bridge-brave</artifactId>
+            <artifactId>micrometer-tracing-bridge-otel</artifactId>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.dataformat</groupId>
diff --git a/cps-application/src/main/resources/application.yml b/cps-application/src/main/resources/application.yml
index e724ef4443..9c8c1ecd5f 100644
--- a/cps-application/src/main/resources/application.yml
+++ b/cps-application/src/main/resources/application.yml
@@ -151,8 +151,23 @@ security:
         username: ${CPS_USERNAME:cpsuser}
         password: ${CPS_PASSWORD:cpsr0cks!}
 
+cps:
+    tracing:
+        sampler:
+            jaeger_remote:
+                endpoint: ${ONAP_OTEL_SAMPLER_JAEGER_REMOTE_ENDPOINT:http://onap-otel-collector:14250}
+        exporter:
+            endpoint: ${ONAP_OTEL_EXPORTER_ENDPOINT:http://onap-otel-collector:4317}
+            protocol: ${ONAP_OTEL_EXPORTER_PROTOCOL:grpc}
+        enabled: ${ONAP_TRACING_ENABLED:false}
+
 # Actuator
 management:
+    tracing:
+        propagation:
+            produce: ${ONAP_PROPAGATOR_PRODUCE:[W3C]}
+        sampling:
+            probability: 1.0
     endpoints:
         web:
             exposure:
@@ -214,3 +229,9 @@ hazelcast:
         kubernetes:
             enabled: ${HAZELCAST_MODE_KUBERNETES_ENABLED:false}
             service-name: ${CPS_NCMP_SERVICE_NAME:"cps-and-ncmp-service"}
+
+otel:
+    exporter:
+        otlp:
+            traces:
+                protocol: ${ONAP_OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:grpc}
\ No newline at end of file
diff --git a/cps-dependencies/pom.xml b/cps-dependencies/pom.xml
index f323cd7077..a50a420b05 100644
--- a/cps-dependencies/pom.xml
+++ b/cps-dependencies/pom.xml
@@ -42,6 +42,7 @@
         <testcontainers.version>1.18.3</testcontainers.version>
         <mapstruct.version>1.4.2.Final</mapstruct.version>
         <jetty-version>11.0.16</jetty-version>
+        <version.opentelemetry-instrumentation-bom>2.1.0-alpha</version.opentelemetry-instrumentation-bom>
     </properties>
 
     <build>
@@ -177,9 +178,18 @@
                 <version>3.7.3</version>
             </dependency>
             <dependency>
-                <groupId>io.micrometer</groupId>
-                <artifactId>micrometer-tracing-bridge-brave</artifactId>
-                <version>1.0.0</version>
+                <groupId>io.opentelemetry.instrumentation</groupId>
+                <artifactId>opentelemetry-instrumentation-bom-alpha</artifactId>
+                <version>${version.opentelemetry-instrumentation-bom}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-bom</artifactId>
+                <version>1.37.0</version>
+                <type>pom</type>
+                <scope>import</scope>
             </dependency>
             <dependency>
                 <groupId>io.swagger.core.v3</groupId>
diff --git a/cps-ncmp-service/pom.xml b/cps-ncmp-service/pom.xml
index 77e13dbae8..55abffc9bf 100644
--- a/cps-ncmp-service/pom.xml
+++ b/cps-ncmp-service/pom.xml
@@ -38,6 +38,22 @@
     </properties>
     <dependencies>
         <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk-extension-jaeger-remote-sampler</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-kafka-clients-2.6</artifactId>
+        </dependency>
+        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
@@ -104,5 +120,13 @@
             <artifactId>spock</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>jakarta.servlet</groupId>
+            <artifactId>jakarta.servlet-api</artifactId>
+        </dependency>
     </dependencies>
 </project>
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfig.java
new file mode 100644
index 0000000000..bcbacbd421
--- /dev/null
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfig.java
@@ -0,0 +1,111 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2024 Nordix Foundation
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.impl.config;
+
+import io.micrometer.observation.ObservationPredicate;
+import io.micrometer.observation.ObservationRegistry;
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
+import io.opentelemetry.sdk.extension.trace.jaeger.sampler.JaegerRemoteSampler;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import java.time.Duration;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.actuate.autoconfigure.observation.ObservationRegistryCustomizer;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.server.observation.ServerRequestObservationContext;
+import org.springframework.util.AntPathMatcher;
+import org.springframework.util.PathMatcher;
+
+@Configuration
+public class OpenTelemetryConfig {
+
+    public static final int JAEGER_REMOTE_SAMPLER_POLLING_INTERVAL_IN_SECOND = 30;
+
+    @Value("${spring.application.name:cps-application}")
+    private String serviceId;
+
+    @Value("${cps.tracing.exporter.endpoint:http://onap-otel-collector:4317}")
+    private String tracingExporterEndpointUrl;
+
+    @Value("${cps.tracing.sampler.jaeger_remote.endpoint:http://onap-otel-collector:14250}")
+    private String jaegerRemoteSamplerUrl;
+
+    /**
+    * OTLP Exporter with Grpc exporter protocol.
+    */
+    @Bean
+    @ConditionalOnExpression(
+        "${cps.tracing.enabled} && 'grpc'.equals('${cps.tracing.exporter.protocol}')")
+    public OtlpGrpcSpanExporter createOtlpExporterGrpc() {
+        return OtlpGrpcSpanExporter.builder().setEndpoint(tracingExporterEndpointUrl).build();
+    }
+
+    /**
+     * OTLP Exporter with HTTP exporter protocol.
+     */
+    @Bean
+    @ConditionalOnExpression(
+        "${cps.tracing.enabled} && 'http'.equals('${cps.tracing.exporter.protocol}')")
+    public OtlpHttpSpanExporter createOtlpExporterHttp() {
+        return OtlpHttpSpanExporter.builder().setEndpoint(tracingExporterEndpointUrl).build();
+    }
+
+    /**
+     * Jaeger Remote Sampler.
+     */
+    @Bean
+    @ConditionalOnProperty("cps.tracing.enabled")
+    public JaegerRemoteSampler createJaegerRemoteSampler() {
+        return JaegerRemoteSampler.builder()
+          .setEndpoint(jaegerRemoteSamplerUrl)
+          .setPollingInterval(Duration.ofSeconds(JAEGER_REMOTE_SAMPLER_POLLING_INTERVAL_IN_SECOND))
+          .setInitialSampler(Sampler.alwaysOff())
+          .setServiceName(serviceId)
+          .build();
+    }
+
+    /**
+   * Excluding /actuator/** endpoints.
+   */
+    @Bean
+    @ConditionalOnProperty("cps.tracing.enabled")
+    ObservationRegistryCustomizer<ObservationRegistry> skipActuatorEndpointsFromObservation() {
+        final PathMatcher pathMatcher = new AntPathMatcher("/");
+        return registry ->
+          registry.observationConfig().observationPredicate(observationPredicate(pathMatcher));
+    }
+
+    /**
+     * Excluding /actuator/** endpoints.
+     */
+    static ObservationPredicate observationPredicate(final PathMatcher pathMatcher) {
+        return (name, context) -> {
+            if (context instanceof ServerRequestObservationContext observationContext) {
+                return !pathMatcher.match("/actuator/**", observationContext.getCarrier().getRequestURI());
+            } else {
+                return true;
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java
index 167df5a98d..cf6f1c5b17 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfig.java
@@ -21,10 +21,14 @@
 package org.onap.cps.ncmp.api.impl.config.kafka;
 
 import io.cloudevents.CloudEvent;
+import io.opentelemetry.instrumentation.kafkaclients.v2_6.TracingConsumerInterceptor;
+import io.opentelemetry.instrumentation.kafkaclients.v2_6.TracingProducerInterceptor;
 import java.time.Duration;
 import java.util.Map;
 import lombok.RequiredArgsConstructor;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.clients.producer.ProducerConfig;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
 import org.springframework.boot.ssl.SslBundles;
 import org.springframework.context.annotation.Bean;
@@ -52,6 +56,9 @@ public class KafkaConfig<T> {
 
     private final KafkaProperties kafkaProperties;
 
+    @Value("${cps.tracing.enabled:false}")
+    private boolean tracingEnabled;
+
     private static final SslBundles NO_SSL = null;
 
     /**
@@ -64,6 +71,10 @@ public class KafkaConfig<T> {
     public ProducerFactory<String, T> legacyEventProducerFactory() {
         final Map<String, Object> producerConfigProperties = kafkaProperties.buildProducerProperties(NO_SSL);
         producerConfigProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
+        if (tracingEnabled) {
+            producerConfigProperties.put(
+                ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingProducerInterceptor.class.getName());
+        }
         return new DefaultKafkaProducerFactory<>(producerConfigProperties);
     }
 
@@ -77,6 +88,10 @@ public class KafkaConfig<T> {
     public ConsumerFactory<String, T> legacyEventConsumerFactory() {
         final Map<String, Object> consumerConfigProperties = kafkaProperties.buildConsumerProperties(NO_SSL);
         consumerConfigProperties.put("spring.deserializer.value.delegate.class", JsonDeserializer.class);
+        if (tracingEnabled) {
+            consumerConfigProperties.put(
+                ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingConsumerInterceptor.class.getName());
+        }
         return new DefaultKafkaConsumerFactory<>(consumerConfigProperties);
     }
 
@@ -90,6 +105,9 @@ public class KafkaConfig<T> {
     public KafkaTemplate<String, T> legacyEventKafkaTemplate() {
         final KafkaTemplate<String, T> kafkaTemplate = new KafkaTemplate<>(legacyEventProducerFactory());
         kafkaTemplate.setConsumerFactory(legacyEventConsumerFactory());
+        if (tracingEnabled) {
+            kafkaTemplate.setObservationEnabled(true);
+        }
         return kafkaTemplate;
     }
 
@@ -104,6 +122,9 @@ public class KafkaConfig<T> {
                 new ConcurrentKafkaListenerContainerFactory<>();
         containerFactory.setConsumerFactory(legacyEventConsumerFactory());
         containerFactory.getContainerProperties().setAuthExceptionRetryInterval(Duration.ofSeconds(10));
+        if (tracingEnabled) {
+            containerFactory.getContainerProperties().setObservationEnabled(true);
+        }
         return containerFactory;
     }
 
@@ -116,6 +137,10 @@ public class KafkaConfig<T> {
     @Bean
     public ProducerFactory<String, CloudEvent> cloudEventProducerFactory() {
         final Map<String, Object> producerConfigProperties = kafkaProperties.buildProducerProperties(NO_SSL);
+        if (tracingEnabled) {
+            producerConfigProperties.put(
+                ProducerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingProducerInterceptor.class.getName());
+        }
         return new DefaultKafkaProducerFactory<>(producerConfigProperties);
     }
 
@@ -128,6 +153,10 @@ public class KafkaConfig<T> {
     @Bean
     public ConsumerFactory<String, CloudEvent> cloudEventConsumerFactory() {
         final Map<String, Object> consumerConfigProperties = kafkaProperties.buildConsumerProperties(NO_SSL);
+        if (tracingEnabled) {
+            consumerConfigProperties.put(
+                ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, TracingConsumerInterceptor.class.getName());
+        }
         return new DefaultKafkaConsumerFactory<>(consumerConfigProperties);
     }
 
@@ -142,6 +171,9 @@ public class KafkaConfig<T> {
         final KafkaTemplate<String, CloudEvent> kafkaTemplate =
             new KafkaTemplate<>(cloudEventProducerFactory());
         kafkaTemplate.setConsumerFactory(cloudEventConsumerFactory());
+        if (tracingEnabled) {
+            kafkaTemplate.setObservationEnabled(true);
+        }
         return kafkaTemplate;
     }
 
@@ -157,6 +189,9 @@ public class KafkaConfig<T> {
                 new ConcurrentKafkaListenerContainerFactory<>();
         containerFactory.setConsumerFactory(cloudEventConsumerFactory());
         containerFactory.getContainerProperties().setAuthExceptionRetryInterval(Duration.ofSeconds(10));
+        if (tracingEnabled) {
+            containerFactory.getContainerProperties().setObservationEnabled(true);
+        }
         return containerFactory;
     }
 
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfigSpec.groovy
new file mode 100644
index 0000000000..07395cf5bc
--- /dev/null
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/OpenTelemetryConfigSpec.groovy
@@ -0,0 +1,81 @@
+/*
+ * ============LICENSE_START=======================================================
+ * Copyright (C) 2024 Nordix Foundation
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.impl.config
+
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter
+import io.opentelemetry.sdk.extension.trace.jaeger.sampler.JaegerRemoteSampler
+import org.spockframework.spring.SpringBean
+import org.springframework.boot.actuate.autoconfigure.observation.ObservationRegistryCustomizer
+import spock.lang.Shared
+import spock.lang.Specification
+
+class OpenTelemetryConfigSpec extends Specification{
+
+    @Shared
+    @SpringBean
+    OpenTelemetryConfig openTelemetryConfig = new OpenTelemetryConfig()
+
+    def setupSpec() {
+        openTelemetryConfig.tracingExporterEndpointUrl="http://tracingExporterEndpointUrl"
+        openTelemetryConfig.jaegerRemoteSamplerUrl="http://jaegerremotesamplerurl"
+        openTelemetryConfig.serviceId ="cps-application"
+    }
+
+    def 'OpenTelemetryConfig Construction.'() {
+        expect: 'the system can create an instance'
+        new OpenTelemetryConfig() != null
+    }
+
+    def  'OTLP Exporter creation with Grpc protocol'(){
+        when: 'an OTLP exporter is created'
+            def result = openTelemetryConfig.createOtlpExporterGrpc()
+        then: 'an OTLP Exporter is created'
+            assert result instanceof OtlpGrpcSpanExporter
+    }
+
+    def  'OTLP Exporter creation with HTTP protocol'(){
+        when: 'an OTLP exporter is created'
+            def result = openTelemetryConfig.createOtlpExporterHttp()
+        then: 'an OTLP Exporter is created'
+            assert result instanceof OtlpHttpSpanExporter
+        and:
+            assert result.builder.endpoint=="http://tracingExporterEndpointUrl"
+    }
+
+    def  'Jaeger Remote Sampler Creation'(){
+        when: 'an OTLP exporter is created'
+            def result = openTelemetryConfig.createJaegerRemoteSampler()
+        then: 'an OTLP Exporter is created'
+            assert result instanceof JaegerRemoteSampler
+        and:
+            assert result.delegate.type=="remoteSampling"
+        and:
+            assert result.delegate.url.toString().startsWith("http://jaegerremotesamplerurl")
+    }
+
+    def  'Skipping Acutator endpoints'(){
+        when: 'an OTLP exporter is created'
+            def result = openTelemetryConfig.skipActuatorEndpointsFromObservation()
+        then: 'an OTLP Exporter is created'
+            assert result instanceof ObservationRegistryCustomizer
+    }
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy
index 16f27d081b..4d3fd6616b 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/config/kafka/KafkaConfigSpec.groovy
@@ -18,7 +18,7 @@
  *  ============LICENSE_END=========================================================
  */
 
-package org.onap.cps.ncmp.api.impl.config.kafka;
+package org.onap.cps.ncmp.api.impl.config.kafka
 
 import io.cloudevents.CloudEvent
 import io.cloudevents.kafka.CloudEventDeserializer
@@ -31,12 +31,14 @@ import org.springframework.boot.test.context.SpringBootTest
 import org.springframework.kafka.core.KafkaTemplate
 import org.springframework.kafka.support.serializer.JsonDeserializer
 import org.springframework.kafka.support.serializer.JsonSerializer
+import org.springframework.test.context.TestPropertySource
 import spock.lang.Shared
 import spock.lang.Specification
 
 @SpringBootTest(classes = [KafkaProperties, KafkaConfig])
 @EnableSharedInjection
 @EnableConfigurationProperties
+@TestPropertySource(properties = ["cps.tracing.enabled=true"])
 class KafkaConfigSpec extends Specification {
 
     @Shared
diff --git a/integration-test/src/test/resources/application.yml b/integration-test/src/test/resources/application.yml
index a4b9ea9c40..58e6287955 100644
--- a/integration-test/src/test/resources/application.yml
+++ b/integration-test/src/test/resources/application.yml
@@ -219,3 +219,9 @@ hazelcast:
     kubernetes:
       enabled: false
       service-name: cps-and-ncmp-service
+
+cps:
+  tracing:
+    enabled: false
+    exporter:
+      protocol: grpc