A1-PMS http Telemetry

Issue-ID: CCSDK-4010
Change-Id: Ib81f246f3e79f49c9361e802b2a73a99f7eb6db9
Signed-off-by: lapentafd <francesco.lapenta@est.tech>

10 files changed, 395 insertions, 8 deletions

diff --git a/a1-policy-management/config/application.yaml b/a1-policy-management/config/application.yaml
index 4f80d2e3..52b70d10 100644
--- a/a1-policy-management/config/application.yaml
+++ b/a1-policy-management/config/application.yaml
@@ -20,6 +20,8 @@
 # ============LICENSE_END=========================================================
 #
 spring:
+  application:
+    name: a1-pms
   profiles:
     active: prod
   main:
@@ -27,6 +29,22 @@ spring:
   aop:
     auto: false
 management:
+  otlp:
+    metrics:
+      export:
+        enabled: false
+  tracing:
+    sampler:
+      jaeger_remote:
+        endpoint: ${ONAP_OTEL_SAMPLER_JAEGER_REMOTE_ENDPOINT:http://jaeger:14250}
+    exporter:
+      endpoint: ${ONAP_OTEL_EXPORTER_ENDPOINT:http://jaeger:4317}
+      protocol: ${ONAP_OTEL_EXPORTER_PROTOCOL:grpc}
+    enabled: ${ONAP_TRACING_ENABLED:false}
+    propagation:
+      produce: ${ONAP_PROPAGATOR_PRODUCE:[W3C]}
+    sampling:
+      probability: 1.0
   endpoints:
     web:
       exposure:
@@ -100,3 +118,8 @@ app:
     accessKeyId: minio
     secretAccessKey: miniostorage
     bucket:
+otel:
+  exporter:
+    otlp:
+      traces:
+        protocol: ${ONAP_OTEL_EXPORTER_OTLP_TRACES_PROTOCOL:grpc}
diff --git a/a1-policy-management/pom.xml b/a1-policy-management/pom.xml
index b9b34a27..c383ba2f 100644
--- a/a1-policy-management/pom.xml
+++ b/a1-policy-management/pom.xml
@@ -215,7 +215,57 @@
             <artifactId>everit-json-schema</artifactId>
             <version>1.14.0</version>
         </dependency>
+        <!-- For Tracing -->
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-tracing-bridge-otel</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-exporter-otlp</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk-extension-jaeger-remote-sampler</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-spring-webflux-5.3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>context-propagation</artifactId>
+            <version>1.0.2</version>
+        </dependency>
+        <!-- For ObservationRegistryCustomizer -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-actuator-autoconfigure</artifactId>
+            <version>3.1.0</version>
+        </dependency>
     </dependencies>
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>io.opentelemetry</groupId>
+                <artifactId>opentelemetry-bom</artifactId>
+                <version>1.38.0</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+            <dependency>
+                <groupId>io.opentelemetry.instrumentation</groupId>
+                <artifactId>opentelemetry-instrumentation-bom-alpha</artifactId>
+                <version>2.4.0-alpha</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
     <build>
         <plugins>
             <plugin>
diff --git a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/AsyncRestClient.java b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/AsyncRestClient.java
index e3d99426..ddcfd068 100644
--- a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/AsyncRestClient.java
+++ b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/AsyncRestClient.java
@@ -3,6 +3,7 @@
  * ONAP : ccsdk oran
  * ======================================================================
  * Copyright (C) 2019-2022 Nordix Foundation. All rights reserved.
+ * Copyright (C) 2024 OpenInfra Foundation Europe. All rights reserved.
  * ======================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,10 +25,13 @@ import io.netty.channel.ChannelOption;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.timeout.ReadTimeoutHandler;
 import io.netty.handler.timeout.WriteTimeoutHandler;
+import io.opentelemetry.instrumentation.spring.webflux.v5_3.SpringWebfluxTelemetry;
 
 import java.lang.invoke.MethodHandles;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ApplicationContextProvider;
+import org.onap.ccsdk.oran.a1policymanagementservice.configuration.OtelConfig;
 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.WebClientConfig.HttpProxyConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,6 +61,7 @@ public class AsyncRestClient {
     private final SslContext sslContext;
     private final HttpProxyConfig httpProxyConfig;
     private final SecurityContext securityContext;
+    private OtelConfig otelConfig = ApplicationContextProvider.getApplicationContext().getBean(OtelConfig.class);
 
     public AsyncRestClient(String baseUrl, @Nullable SslContext sslContext, @Nullable HttpProxyConfig httpProxyConfig,
             SecurityContext securityContext) {
@@ -205,13 +210,19 @@ public class AsyncRestClient {
             return Mono.just(resp);
         });
 
-        return WebClient.builder() //
-                .clientConnector(new ReactorClientHttpConnector(httpClient)) //
-                .baseUrl(baseUrl) //
-                .exchangeStrategies(exchangeStrategies) //
-                .filter(reqLogger) //
-                .filter(respLogger) //
-                .build();
+        WebClient.Builder webClientBuilder = WebClient.builder()
+                .clientConnector(new ReactorClientHttpConnector(httpClient))
+                .baseUrl(baseUrl)
+                .exchangeStrategies(exchangeStrategies)
+                .filter(reqLogger)
+                .filter(respLogger);
+
+        if (otelConfig.isTracingEnabled()) {
+            SpringWebfluxTelemetry webfluxTelemetry = ApplicationContextProvider.getApplicationContext().getBean(SpringWebfluxTelemetry.class);
+            webClientBuilder.filters(webfluxTelemetry::addClientTracingFilter);
+        }
+
+        return webClientBuilder.build();
     }
 
     private WebClient getWebClient() {
diff --git a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/ApplicationContextProvider.java b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/ApplicationContextProvider.java
new file mode 100644
index 00000000..a7130201
--- /dev/null
+++ b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/ApplicationContextProvider.java
@@ -0,0 +1,39 @@
+/*-
+ * ========================LICENSE_START=================================
+ * ONAP : ccsdk oran
+ * ======================================================================
+ * Copyright (C) 2024 OpenInfra Foundation Europe. All rights reserved.
+ * ======================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ========================LICENSE_END===================================
+ */
+package org.onap.ccsdk.oran.a1policymanagementservice.configuration;
+
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ApplicationContextProvider implements ApplicationContextAware {
+    private static ApplicationContext context;
+
+    @Override
+    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
+        context = applicationContext;
+    }
+
+    public static ApplicationContext getApplicationContext() {
+        return context;
+    }
+}
\ No newline at end of file
diff --git a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfig.java b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfig.java
new file mode 100644
index 00000000..a66bc062
--- /dev/null
+++ b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfig.java
@@ -0,0 +1,125 @@
+/*-
+ * ========================LICENSE_START=================================
+ * ONAP : ccsdk oran
+ * ======================================================================
+ * Copyright (C) 2024 OpenInfra Foundation Europe. All rights reserved.
+ * ======================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ========================LICENSE_END===================================
+ */
+package org.onap.ccsdk.oran.a1policymanagementservice.configuration;
+
+import io.micrometer.observation.ObservationPredicate;
+import io.micrometer.observation.ObservationRegistry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
+import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter;
+import io.opentelemetry.instrumentation.spring.webflux.v5_3.SpringWebfluxTelemetry;
+import io.opentelemetry.sdk.extension.trace.jaeger.sampler.JaegerRemoteSampler;
+import io.opentelemetry.sdk.trace.samplers.Sampler;
+import lombok.Getter;
+import reactor.core.publisher.Hooks;
+
+import java.time.Duration;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.server.observation.ServerRequestObservationContext;
+import org.springframework.boot.actuate.autoconfigure.observation.ObservationRegistryCustomizer;
+import org.springframework.util.AntPathMatcher;
+import org.springframework.util.PathMatcher;
+
+@Configuration
+@ComponentScan(basePackages = {"org.onap.ccsdk.oran.a1policymanagementservice"})
+public class OtelConfig {
+    private static final Logger logger = LoggerFactory.getLogger(OtelConfig.class);
+
+    public static final int JAEGER_REMOTE_SAMPLER_POLLING_INTERVAL_IN_SECOND = 30;
+
+    @Value("${spring.application.name}")
+    private String serviceId;
+
+    @Value("${management.tracing.exporter.endpoint}")
+    private String tracingExporterEndpointUrl;
+
+    @Value("${management.tracing.sampler.jaeger-remote.endpoint}")
+    private String jaegerRemoteSamplerUrl;
+
+    @Value("${management.tracing.exporter.protocol}")
+    private String tracingProtocol;
+
+    @Getter
+    @Value("${management.tracing.enabled}")
+    private boolean tracingEnabled;
+
+    @PostConstruct
+    public void checkTracingConfig() {
+        logger.info("Application Yaml Tracing Enabled: " + tracingEnabled);
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "management.tracing", name = "enabled", havingValue = "true", matchIfMissing = false)
+    @ConditionalOnExpression("'grpc'.equals('${management.tracing.exporter.protocol}')")
+    public OtlpGrpcSpanExporter otlpExporterGrpc() {
+        return OtlpGrpcSpanExporter.builder().setEndpoint(tracingExporterEndpointUrl).build();
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "management.tracing", name = "enabled", havingValue = "true", matchIfMissing = false)
+    @ConditionalOnExpression("'http'.equals('${management.tracing.exporter.protocol}')")
+    public OtlpHttpSpanExporter otlpExporterHttp() {
+        return OtlpHttpSpanExporter.builder().setEndpoint(tracingExporterEndpointUrl).build();
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "management.tracing", name = "enabled", havingValue = "true", matchIfMissing = false)
+    public JaegerRemoteSampler jaegerRemoteSampler() {
+        return JaegerRemoteSampler.builder().setEndpoint(jaegerRemoteSamplerUrl)
+                .setPollingInterval(Duration.ofSeconds(JAEGER_REMOTE_SAMPLER_POLLING_INTERVAL_IN_SECOND))
+                .setInitialSampler(Sampler.alwaysOff()).setServiceName(serviceId).build();
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "management.tracing", name = "enabled", havingValue = "true", matchIfMissing = false)
+    public SpringWebfluxTelemetry webfluxTelemetry (OpenTelemetry openTelemetry) {
+        //enables automatic context propagation to ThreadLocals used by FLUX and MONO operators
+        Hooks.enableAutomaticContextPropagation();
+        return SpringWebfluxTelemetry.builder(openTelemetry).build();
+    }
+
+    @Bean
+    @ConditionalOnProperty(prefix = "management.tracing", name = "enabled", havingValue = "true", matchIfMissing = false)
+    ObservationRegistryCustomizer<ObservationRegistry> skipActuatorEndpointsFromObservation() {
+        PathMatcher pathMatcher = new AntPathMatcher("/");
+        return registry ->
+            registry.observationConfig().observationPredicate(observationPredicate(pathMatcher));
+    }
+
+    static ObservationPredicate observationPredicate(PathMatcher pathMatcher) {
+        return (name, context) -> {
+            if (context instanceof ServerRequestObservationContext observationContext) {
+                return !pathMatcher.match("/actuator/**", observationContext.getCarrier().getRequestURI());
+            } else {
+                return false;
+            }
+        };
+    }
+}
diff --git a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/authorization/AuthorizationCheck.java b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/authorization/AuthorizationCheck.java
index 0f376c06..5ad50688 100644
--- a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/authorization/AuthorizationCheck.java
+++ b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/authorization/AuthorizationCheck.java
@@ -36,12 +36,14 @@ import org.onap.ccsdk.oran.a1policymanagementservice.repository.Policy;
 import org.onap.ccsdk.oran.a1policymanagementservice.repository.PolicyType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.DependsOn;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Component;
 
 import reactor.core.publisher.Mono;
 
 @Component
+@DependsOn("applicationContextProvider")
 public class AuthorizationCheck {
 
     private final ApplicationConfig applicationConfig;
diff --git a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/v2/ServiceController.java b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/v2/ServiceController.java
index da157db0..5c92b543 100644
--- a/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/v2/ServiceController.java
+++ b/a1-policy-management/src/main/java/org/onap/ccsdk/oran/a1policymanagementservice/controllers/v2/ServiceController.java
@@ -36,7 +36,6 @@ import org.onap.ccsdk.oran.a1policymanagementservice.repository.Services;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ServerWebExchange;
 import reactor.core.publisher.Mono;
diff --git a/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/A1ClientFactoryTest.java b/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/A1ClientFactoryTest.java
index a23540e5..b3c77355 100644
--- a/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/A1ClientFactoryTest.java
+++ b/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/clients/A1ClientFactoryTest.java
@@ -41,11 +41,25 @@ import org.onap.ccsdk.oran.a1policymanagementservice.configuration.ControllerCon
 import org.onap.ccsdk.oran.a1policymanagementservice.configuration.RicConfig;
 import org.onap.ccsdk.oran.a1policymanagementservice.exceptions.ServiceException;
 import org.onap.ccsdk.oran.a1policymanagementservice.repository.Ric;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
 
 import org.onap.ccsdk.oran.a1policymanagementservice.utils.MockA1Client;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.TestPropertySource;
+
 import reactor.core.publisher.Mono;
 import reactor.test.StepVerifier;
 
+@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
+@TestPropertySource(properties = { //
+    "management.tracing.enabled=false",
+    "server.ssl.key-store=./config/keystore.jks", //
+    "app.webclient.trust-store=./config/truststore.jks", //
+    "app.webclient.trust-store-used=true", //
+    "app.vardata-directory=/tmp/pmstest", //
+    "app.filepath=", //
+    "app.s3.bucket=" // If this is set, S3 will be used to store data.
+})
 @ExtendWith(MockitoExtension.class)
 class A1ClientFactoryTest {
     private static final String RIC_NAME = "Name";
diff --git a/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfigTest.java b/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfigTest.java
new file mode 100644
index 00000000..7c04de20
--- /dev/null
+++ b/a1-policy-management/src/test/java/org/onap/ccsdk/oran/a1policymanagementservice/configuration/OtelConfigTest.java
@@ -0,0 +1,122 @@
+/*-
+ * ========================LICENSE_START=================================
+ * ONAP : ccsdk oran
+ * ======================================================================
+ * Copyright (C) 2024 OpenInfra Foundation Europe. All rights reserved.
+ * ======================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ========================LICENSE_END===================================
+ */
+package org.onap.ccsdk.oran.a1policymanagementservice.configuration;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
+import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
+import jakarta.servlet.http.HttpServletRequest;
+import java.util.Objects;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.context.SpringBootTest.WebEnvironment;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.http.server.observation.ServerRequestObservationContext;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.util.AntPathMatcher;
+
+@SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT)
+@TestPropertySource(properties = { //
+    "server.ssl.key-store=./config/keystore.jks",
+    "app.webclient.trust-store=./config/truststore.jks",
+    "app.webclient.trust-store-used=true",
+    "app.vardata-directory=/tmp/pmstest",
+    "app.filepath=",
+    "app.s3.bucket=",
+    "spring.application.name=a1-pms",
+    "management.tracing.enabled=true",
+    "management.tracing.exporter.protocol=grpc",
+    "management.tracing.sampler.jaeger_remote.endpoint=http://127.0.0.1:14250",
+    "management.tracing.propagator.type=W3C"
+})
+@AutoConfigureObservability
+class OtelConfigTest {
+
+    @Autowired private ApplicationContext context;
+
+    @Autowired OtelConfig otelConfig;
+
+    @Autowired ObservationRegistry observationRegistry;
+
+    @Bean
+    OpenTelemetry openTelemetry() {
+        return AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk();
+    }
+
+    @Test
+    void otlpExporterGrpc() {
+        assertNotNull(otelConfig);
+        assertNotNull(otelConfig.otlpExporterGrpc());
+    }
+
+    @Test
+    void otlpExporterHttpNotActive() {
+        assertNotNull(otelConfig);
+        assertThrows(BeansException.class, () -> context.getBean(OtlpHttpSpanExporter.class));
+    }
+
+    @Test
+    void jaegerRemoteSampler() {
+        assertNotNull(otelConfig);
+        assertNotNull(otelConfig.jaegerRemoteSampler());
+    }
+
+    @Test
+    void skipActuatorEndpointsFromObservation() {
+        assertNotNull(otelConfig);
+        var actuatorCustomizer = otelConfig.skipActuatorEndpointsFromObservation();
+        assertNotNull(actuatorCustomizer);
+        Observation.Scope otelScope = Observation.Scope.NOOP;
+        observationRegistry.setCurrentObservationScope(otelScope);
+        Objects.requireNonNull(observationRegistry.getCurrentObservation()).start();
+    }
+
+    @Test
+    void observationPredicate() {
+        var antPathMatcher = new AntPathMatcher("/");
+        ServerRequestObservationContext serverRequestObservationContext =
+            mock(ServerRequestObservationContext.class);
+        HttpServletRequest httpServletRequest = mock(HttpServletRequest.class);
+        when(httpServletRequest.getRequestURI()).thenReturn("/actuator/health");
+        when(serverRequestObservationContext.getCarrier()).thenReturn(httpServletRequest);
+        boolean result =
+            OtelConfig.observationPredicate(antPathMatcher)
+                .test("anything", serverRequestObservationContext);
+        assertFalse(result);
+        when(httpServletRequest.getRequestURI()).thenReturn("/api/v1/anything");
+        result =
+            OtelConfig.observationPredicate(antPathMatcher)
+                .test("anything", serverRequestObservationContext);
+        assertTrue(result);
+    }
+}
diff --git a/csit/scripts/healthcheck/test/health_check.sh b/csit/scripts/healthcheck/test/health_check.sh
index ad23632f..0be6489f 100755
--- a/csit/scripts/healthcheck/test/health_check.sh
+++ b/csit/scripts/healthcheck/test/health_check.sh
@@ -48,6 +48,8 @@ checkStatus(){
         fi
     done
     echo -e "$i sec: $4 is NOT alive!\n"
+    echo "Capturing docker logs..."
+    docker-compose logs
     exit -1
 }