Dmi plugin watchdog cheking aliveness

- Add capability of GET request into DmiRestClient
- Add watchdog job to check aliveness of dmi plugins
- DmiPluginStatus enum as UP or DOWN
- Add unit tests for the new function in dmi rest client
- Add unit tests for dmi watchdog

Issue-ID: CPS-1856
Change-Id: Ic38a96f0485a0bfe1b6af5bb2f57f6119d8fa563
Signed-off-by: halil.cakal <halil.cakal@est.tech>

8 files changed, 242 insertions, 11 deletions

diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java
index 28e09ac4bc..7066f80165 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/client/DmiRestClient.java
@@ -21,10 +21,13 @@
 
 package org.onap.cps.ncmp.api.impl.client;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.ncmp.api.impl.config.NcmpConfiguration.DmiProperties;
 import org.onap.cps.ncmp.api.impl.exception.HttpClientRequestException;
 import org.onap.cps.ncmp.api.impl.operations.OperationType;
+import org.onap.cps.ncmp.api.impl.trustlevel.dmiavailability.DmiPluginStatus;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
@@ -35,6 +38,7 @@ import org.springframework.web.client.RestTemplate;
 
 @Component
 @AllArgsConstructor
+@Slf4j
 public class DmiRestClient {
 
     private RestTemplate restTemplate;
@@ -60,6 +64,28 @@ public class DmiRestClient {
         }
     }
 
+    /**
+     * Sends GET operation to DMI plugin's health check URL.
+     *
+     * @param       dmiPluginBaseUrl the base URL of the dmi-plugin
+     * @return      DmiPluginStatus as UP or DOWN
+     */
+    public DmiPluginStatus getDmiPluginStatus(final String dmiPluginBaseUrl) {
+        try {
+            final HttpEntity<Object> httpHeaders = new HttpEntity<>(configureHttpHeaders(new HttpHeaders()));
+            final JsonNode dmiPluginHealthStatus = restTemplate.getForObject(dmiPluginBaseUrl + "/manage/health",
+                    JsonNode.class, httpHeaders);
+            if (dmiPluginHealthStatus != null) {
+                if (dmiPluginHealthStatus.get("status").asText().equals("UP")) {
+                    return DmiPluginStatus.UP;
+                }
+            }
+        } catch (final Exception exception) {
+            log.warn("Could not send request for health check since {}", exception.getMessage());
+        }
+        return DmiPluginStatus.DOWN;
+    }
+
     private HttpHeaders configureHttpHeaders(final HttpHeaders httpHeaders) {
         if (dmiProperties.isDmiBasicAuthEnabled()) {
             httpHeaders.setBasicAuth(dmiProperties.getAuthUsername(), dmiProperties.getAuthPassword());
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/embeddedcache/TrustLevelCacheConfig.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/embeddedcache/TrustLevelCacheConfig.java
index 66f61906da..ebe99057d7 100644
--- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/embeddedcache/TrustLevelCacheConfig.java
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/config/embeddedcache/TrustLevelCacheConfig.java
@@ -32,24 +32,25 @@ import org.springframework.context.annotation.Configuration;
 @Configuration
 public class TrustLevelCacheConfig extends HazelcastCacheConfig {
 
-    private static final SetConfig untrustworthyCmHandlesSetConfig = createSetConfig("untrustworthyCmHandlesSetConfig");
+    private static final SetConfig untrustworthyCmHandlesSetConfig =
+            createSetConfig("untrustworthyCmHandlesSetConfig");
+
+    private static final MapConfig trustLevelPerDmiPluginCacheConfig =
+            createMapConfig("trustLevelPerDmiPluginCacheConfig");
 
     /**
-     * Untrustworthy cmhandle set instance.
+     * Distributed collection of untrustworthy cm handles.
      *
-     * @return instance of distributed set of untrustworthy cmhandles.
+     * @return instance of distributed set of untrustworthy cm handles.
      */
     @Bean
     public ISet<String> untrustworthyCmHandlesSet() {
-        return createHazelcastInstance("untrustworthyCmHandlesSet", untrustworthyCmHandlesSetConfig).getSet(
-                "untrustworthyCmHandlesSet");
+        return createHazelcastInstance("untrustworthyCmHandlesSet",
+                untrustworthyCmHandlesSetConfig).getSet("untrustworthyCmHandlesSet");
     }
 
-    private static final MapConfig trustLevelPerDmiPluginCacheConfig =
-            createMapConfig("trustLevelPerDmiPluginCacheConfig");
-
     /**
-     * Distributed instance of trust level cache that contains dmi-plugin name by trust level.
+     * Distributed instance of trust level cache containing the trust level per dmi plugin service(name).
      *
      * @return configured map of dmi-plugin name as keys to trust-level for values.
      */
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDog.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDog.java
new file mode 100644
index 0000000000..dac32aa736
--- /dev/null
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDog.java
@@ -0,0 +1,60 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2023 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.impl.trustlevel.dmiavailability;
+
+import com.hazelcast.map.IMap;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.onap.cps.ncmp.api.impl.client.DmiRestClient;
+import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@RequiredArgsConstructor
+@Service
+public class DMiPluginWatchDog {
+
+    private final IMap<String, TrustLevel> trustLevelPerDmiPlugin;
+
+    private final DmiRestClient dmiRestClient;
+
+    /**
+     * Monitors the aliveness of DMI plugins by this watchdog.
+     * This method periodically checks the health and status of each DMI plugin to ensure that
+     * they are functioning properly. If a plugin is found to be unresponsive or in an
+     * unhealthy state, the cache will be updated with the latest status.
+     * The @fixedDelayString is the time interval, in milliseconds, between consecutive aliveness checks.
+     */
+    @Scheduled(fixedDelayString = "${ncmp.timers.trust-evel.dmi-availability-watchdog-ms:30000}")
+    public void watchDmiPluginAliveness() {
+        trustLevelPerDmiPlugin.keySet().forEach((dmiPluginName) -> {
+            final DmiPluginStatus dmiPluginStatus = dmiRestClient.getDmiPluginStatus(dmiPluginName);
+            log.debug("Trust level for dmi-plugin: {} is {}", dmiPluginName, dmiPluginStatus.toString());
+            if (DmiPluginStatus.UP.equals(dmiPluginStatus)) {
+                trustLevelPerDmiPlugin.put(dmiPluginName, TrustLevel.COMPLETE);
+            } else {
+                trustLevelPerDmiPlugin.put(dmiPluginName, TrustLevel.NONE);
+            }
+        });
+    }
+
+}
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DmiPluginStatus.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DmiPluginStatus.java
new file mode 100644
index 0000000000..352d36f942
--- /dev/null
+++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DmiPluginStatus.java
@@ -0,0 +1,25 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2023 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.impl.trustlevel.dmiavailability;
+
+public enum DmiPluginStatus {
+    UP, DOWN;
+}
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy
index 0d03fd9acf..80c0a27bf7 100644
--- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/client/DmiRestClientSpec.groovy
@@ -21,8 +21,13 @@
 
 package org.onap.cps.ncmp.api.impl.client
 
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.node.ObjectNode
 import org.onap.cps.ncmp.api.impl.config.NcmpConfiguration
 import org.onap.cps.ncmp.api.impl.exception.HttpClientRequestException
+import org.onap.cps.ncmp.api.impl.trustlevel.dmiavailability.DmiPluginStatus
+import org.onap.cps.ncmp.utils.TestUtils
 import org.spockframework.spring.SpringBean
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.boot.test.context.SpringBootTest
@@ -40,7 +45,7 @@ import static org.onap.cps.ncmp.api.impl.operations.OperationType.PATCH
 import static org.onap.cps.ncmp.api.impl.operations.OperationType.CREATE
 
 @SpringBootTest
-@ContextConfiguration(classes = [NcmpConfiguration.DmiProperties, DmiRestClient])
+@ContextConfiguration(classes = [NcmpConfiguration.DmiProperties, DmiRestClient, ObjectMapper])
 class DmiRestClientSpec extends Specification {
 
     @SpringBean
@@ -48,8 +53,11 @@ class DmiRestClientSpec extends Specification {
 
     @Autowired
     DmiRestClient objectUnderTest
-    def resourceUrl = 'some url'
 
+    @Autowired
+    ObjectMapper objectMapper
+
+    def resourceUrl = 'some url'
     def mockResponseEntity = Mock(ResponseEntity)
     def dmiProperties = new NcmpConfiguration.DmiProperties()
 
@@ -85,6 +93,36 @@ class DmiRestClientSpec extends Specification {
             operation << [CREATE, READ, PATCH]
     }
 
+    def 'Get dmi plugin health status #scenario'() {
+        given: 'a health check response data as jsonNode'
+            def dmiPluginHealthCheckResponseJsonData = TestUtils.getResourceFileContent('dmiPluginHealthCheckResponse.json')
+            def jsonNode = objectMapper.readValue(dmiPluginHealthCheckResponseJsonData, JsonNode.class)
+            ((ObjectNode) jsonNode).put('status', dmiAliveness);
+        and: 'the rest template return a valid json node'
+            mockRestTemplate.getForObject(*_) >> {jsonNode}
+        when: 'get aliveness of the dmi plugin'
+            def result = objectUnderTest.getDmiPluginStatus(resourceUrl)
+        then: 'return value is equal to result of rest template call'
+            result == expectedResult
+        where: 'the following dmi aliveness are being used'
+            scenario             | dmiAliveness || expectedResult
+            'dmi plugin is UP'   | 'UP'         || DmiPluginStatus.UP
+            'dmi plugin is DOWN' | 'DOWN'       || DmiPluginStatus.DOWN
+    }
+
+    def 'Failing to get dmi plugin health status #scenario'() {
+        given: 'the rest template return null'
+            mockRestTemplate.getForObject(*_) >> {getResponse}
+        when: 'get aliveness of the dmi plugin'
+            def result = objectUnderTest.getDmiPluginStatus(resourceUrl)
+        then: 'return value is equal to result of rest template call'
+            result == expectedResult
+        where: 'the following dmi responses are being used'
+            scenario                        | getResponse                  || expectedResult
+            'get response is null'          | null                         || DmiPluginStatus.DOWN
+            'get response throws exception' | {throw new Exception()}      || DmiPluginStatus.DOWN
+    }
+
     def 'Basic auth header #scenario'() {
         when: 'Specific dmi properties are provided'
             dmiProperties.dmiBasicAuthEnabled = authEnabled
diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDogSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDogSpec.groovy
new file mode 100644
index 0000000000..af546b7f5e
--- /dev/null
+++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/trustlevel/dmiavailability/DMiPluginWatchDogSpec.groovy
@@ -0,0 +1,51 @@
+/*
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2023 Nordix Foundation
+ *  ================================================================================
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *  ============LICENSE_END=========================================================
+ */
+
+package org.onap.cps.ncmp.api.impl.trustlevel.dmiavailability
+
+import com.hazelcast.map.IMap
+import org.onap.cps.ncmp.api.impl.client.DmiRestClient
+import org.onap.cps.ncmp.api.impl.trustlevel.TrustLevel
+import spock.lang.Specification
+
+class DMiPluginWatchDogSpec extends Specification {
+
+
+    def mockTrustLevelPerDmiPlugin = Mock(IMap<String, TrustLevel>)
+    def mockDmiRestClient = Mock(DmiRestClient)
+    def objectUnderTest = new DMiPluginWatchDog(mockTrustLevelPerDmiPlugin, mockDmiRestClient)
+
+
+    def 'watch dmi plugin aliveness'() {
+        given: 'the dmi client returns aliveness for #dmi1Status'
+            mockDmiRestClient.getDmiPluginStatus('dmi1') >> dmi1Status
+        and: 'trust level cache returns dmi1'
+            mockTrustLevelPerDmiPlugin.keySet() >> {['dmi1'] as Set}
+        when: 'watch dog started'
+            objectUnderTest.watchDmiPluginAliveness()
+        then: 'trust level cache has been populated with #dmi1TrustLevel for dmi1'
+            1 * mockTrustLevelPerDmiPlugin.put('dmi1', dmi1TrustLevel)
+        where: 'the following parameter are used'
+            scenario                  | dmi1Status              || dmi1TrustLevel
+            'dmi1 is UP'              | DmiPluginStatus.UP      || TrustLevel.COMPLETE
+            'dmi1 is DOWN'            | DmiPluginStatus.DOWN    || TrustLevel.NONE
+    }
+
+}
diff --git a/cps-ncmp-service/src/test/resources/application.yml b/cps-ncmp-service/src/test/resources/application.yml
index 6e7577b1a8..a4bb4e8124 100644
--- a/cps-ncmp-service/src/test/resources/application.yml
+++ b/cps-ncmp-service/src/test/resources/application.yml
@@ -42,6 +42,9 @@ ncmp:
             enabled: true
         api:
             base-path: dmi
+    timers:
+        trust-level:
+            dmi-availability-watchdog-ms: 30000
 
     modules-sync-watchdog:
         async-executor:
diff --git a/cps-ncmp-service/src/test/resources/dmiPluginHealthCheckResponse.json b/cps-ncmp-service/src/test/resources/dmiPluginHealthCheckResponse.json
new file mode 100644
index 0000000000..753222005e
--- /dev/null
+++ b/cps-ncmp-service/src/test/resources/dmiPluginHealthCheckResponse.json
@@ -0,0 +1,27 @@
+{
+  "status": "UP",
+  "components": {
+    "diskSpace": {
+      "status": "UP",
+      "details": {
+        "total": 269490393088,
+        "free": 228467286016,
+        "threshold": 10485760,
+        "exists": true
+      }
+    },
+    "livenessState": {
+      "status": "UP"
+    },
+    "ping": {
+      "status": "UP"
+    },
+    "readinessState": {
+      "status": "UP"
+    }
+  },
+  "groups": [
+    "liveness",
+    "readiness"
+  ]
+}
\ No newline at end of file