Merge "Support for Retry and Timeout handling on ParticipantStatus"

9 files changed, 533 insertions, 57 deletions

diff --git a/models/src/main/java/org/onap/policy/clamp/controlloop/models/controlloop/concepts/ParticipantHealthStatus.java b/models/src/main/java/org/onap/policy/clamp/controlloop/models/controlloop/concepts/ParticipantHealthStatus.java
index 0cf41c9cd..e7c5fecda 100644
--- a/models/src/main/java/org/onap/policy/clamp/controlloop/models/controlloop/concepts/ParticipantHealthStatus.java
+++ b/models/src/main/java/org/onap/policy/clamp/controlloop/models/controlloop/concepts/ParticipantHealthStatus.java
@@ -43,5 +43,10 @@ public enum ParticipantHealthStatus {
     /**
      * The health status of the Participant is unknown.
      */
-    UNKNOWN
+    UNKNOWN,
+
+    /**
+     * The health status of the Participant is off line.
+     */
+    OFF_LINE
 }
diff --git a/models/src/main/java/org/onap/policy/clamp/controlloop/models/messages/dmaap/participant/ParticipantRegister.java b/models/src/main/java/org/onap/policy/clamp/controlloop/models/messages/dmaap/participant/ParticipantRegister.java
index 7319d99db..af0149189 100644
--- a/models/src/main/java/org/onap/policy/clamp/controlloop/models/messages/dmaap/participant/ParticipantRegister.java
+++ b/models/src/main/java/org/onap/policy/clamp/controlloop/models/messages/dmaap/participant/ParticipantRegister.java
@@ -23,10 +23,6 @@ package org.onap.policy.clamp.controlloop.models.messages.dmaap.participant;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
-import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoops;
-import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantHealthStatus;
-import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantState;
-import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantStatistics;
 
 /**
  * Class to represent the PARTICIPANT_REGISTER message that all the participants send to control loop runtime.
diff --git a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounter.java b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounter.java
new file mode 100644
index 000000000..2151dc143
--- /dev/null
+++ b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounter.java
@@ -0,0 +1,91 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.clamp.controlloop.runtime.supervision;
+
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import lombok.Getter;
+import lombok.Setter;
+
+public class HandleCounter<K> {
+    @Getter
+    @Setter
+    private long maxWaitMs;
+
+    @Getter
+    @Setter
+    private int maxRetryCount;
+
+    private Map<K, Integer> mapCounter = new HashMap<>();
+    private Set<K> mapFault = new HashSet<>();
+    private Map<K, Long> mapTimer = new HashMap<>();
+
+    public long getDuration(K id) {
+        mapTimer.putIfAbsent(id, getEpochMilli());
+        return getEpochMilli() - mapTimer.get(id);
+    }
+
+    /**
+     * Reset timer and clear counter and fault by id.
+     *
+     * @param id the id
+     */
+    public void clear(K id) {
+        mapFault.remove(id);
+        mapCounter.put(id, 0);
+        mapTimer.put(id, getEpochMilli());
+    }
+
+    public void setFault(K id) {
+        mapCounter.put(id, 0);
+        mapFault.add(id);
+    }
+
+    /**
+     * Increment RetryCount by id e return true if minor or equal of maxRetryCount.
+     *
+     * @param id the identifier
+     * @return false if count is major of maxRetryCount
+     */
+    public boolean count(K id) {
+        int counter = mapCounter.getOrDefault(id, 0) + 1;
+        if (counter <= maxRetryCount) {
+            mapCounter.put(id, counter);
+            return true;
+        }
+        return false;
+    }
+
+    public boolean isFault(K id) {
+        return mapFault.contains(id);
+    }
+
+    public int getCounter(K id) {
+        return mapCounter.getOrDefault(id, 0);
+    }
+
+    protected long getEpochMilli() {
+        return Instant.now().toEpochMilli();
+    }
+}
diff --git a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspect.java b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspect.java
index 293b5d5da..d0d18ab1a 100644
--- a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspect.java
+++ b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspect.java
@@ -28,6 +28,8 @@ import java.util.concurrent.TimeUnit;
 import lombok.RequiredArgsConstructor;
 import org.aspectj.lang.annotation.After;
 import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Before;
+import org.onap.policy.clamp.controlloop.models.messages.dmaap.participant.ParticipantStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.scheduling.annotation.Scheduled;
@@ -64,6 +66,11 @@ public class SupervisionAspect implements Closeable {
         }
     }
 
+    @Before("@annotation(MessageIntercept) && args(participantStatusMessage,..)")
+    public void handleParticipantStatus(ParticipantStatus participantStatusMessage) {
+        executor.execute(() -> supervisionScanner.handleParticipantStatus(participantStatusMessage.getParticipantId()));
+    }
+
     @Override
     public void close() throws IOException {
         executor.shutdown();
diff --git a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScanner.java b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScanner.java
index b360f6703..7be407c3f 100644
--- a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScanner.java
+++ b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScanner.java
@@ -20,19 +20,18 @@
 
 package org.onap.policy.clamp.controlloop.runtime.supervision;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import lombok.Getter;
-import lombok.Setter;
+import java.util.List;
 import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoop;
 import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoopElement;
 import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoopState;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.Participant;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantHealthStatus;
 import org.onap.policy.clamp.controlloop.models.controlloop.persistence.provider.ControlLoopProvider;
+import org.onap.policy.clamp.controlloop.models.controlloop.persistence.provider.ParticipantProvider;
 import org.onap.policy.clamp.controlloop.runtime.main.parameters.ClRuntimeParameterGroup;
 import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ControlLoopStateChangePublisher;
 import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ControlLoopUpdatePublisher;
+import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ParticipantStatusReqPublisher;
 import org.onap.policy.models.base.PfModelException;
 import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier;
 import org.slf4j.Logger;
@@ -46,67 +45,49 @@ import org.springframework.stereotype.Component;
 public class SupervisionScanner {
     private static final Logger LOGGER = LoggerFactory.getLogger(SupervisionScanner.class);
 
-    @Getter
-    @Setter
-    static class HandleCounter {
-        private int maxRetryCount;
-        private long maxWaitMs;
-        private Map<ToscaConceptIdentifier, Integer> mapCounter = new HashMap<>();
-        private Set<ToscaConceptIdentifier> mapFault = new HashSet<>();
-
-        public void clear(ToscaConceptIdentifier id) {
-            mapCounter.put(id, 0);
-            mapFault.remove(id);
-        }
-
-        public void setFault(ToscaConceptIdentifier id) {
-            mapCounter.put(id, 0);
-            mapFault.add(id);
-        }
-
-        public boolean count(ToscaConceptIdentifier id) {
-            int counter = mapCounter.getOrDefault(id, 0) + 1;
-            if (counter <= maxRetryCount) {
-                mapCounter.put(id, counter);
-                return true;
-            }
-            return false;
-        }
-
-        public boolean isFault(ToscaConceptIdentifier id) {
-            return mapFault.contains(id);
-        }
-
-        public int getCounter(ToscaConceptIdentifier id) {
-            return mapCounter.getOrDefault(id, 0);
-        }
-    }
-
-    private HandleCounter stateChange = new HandleCounter();
+    private HandleCounter<ToscaConceptIdentifier> controlLoopCounter = new HandleCounter<>();
+    private HandleCounter<ToscaConceptIdentifier> participantCounter = new HandleCounter<>();
 
     private final ControlLoopProvider controlLoopProvider;
     private final ControlLoopStateChangePublisher controlLoopStateChangePublisher;
     private final ControlLoopUpdatePublisher controlLoopUpdatePublisher;
+    private final ParticipantProvider participantProvider;
+    private final ParticipantStatusReqPublisher participantStatusReqPublisher;
+
+    private final long maxMessageAgeMs;
 
     /**
      * Constructor for instantiating SupervisionScanner.
      *
      * @param controlLoopProvider the provider to use to read control loops from the database
-     * @param controlLoopStateChangePublisher the ControlLoopStateChange Publisher
+     * @param controlLoopStateChangePublisher the ControlLoop StateChange Publisher
+     * @param controlLoopUpdatePublisher the ControlLoopUpdate Publisher
+     * @param participantProvider the Participant Provider
+     * @param participantStatusReqPublisher the Participant StatusReq Publisher
      * @param clRuntimeParameterGroup the parameters for the control loop runtime
      */
     public SupervisionScanner(final ControlLoopProvider controlLoopProvider,
             final ControlLoopStateChangePublisher controlLoopStateChangePublisher,
-            ControlLoopUpdatePublisher controlLoopUpdatePublisher,
+            ControlLoopUpdatePublisher controlLoopUpdatePublisher, ParticipantProvider participantProvider,
+            ParticipantStatusReqPublisher participantStatusReqPublisher,
             final ClRuntimeParameterGroup clRuntimeParameterGroup) {
         this.controlLoopProvider = controlLoopProvider;
         this.controlLoopStateChangePublisher = controlLoopStateChangePublisher;
         this.controlLoopUpdatePublisher = controlLoopUpdatePublisher;
+        this.participantProvider = participantProvider;
+        this.participantStatusReqPublisher = participantStatusReqPublisher;
+
+        controlLoopCounter.setMaxRetryCount(
+                clRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().getMaxRetryCount());
+        controlLoopCounter
+                .setMaxWaitMs(clRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().getMaxWaitMs());
 
-        stateChange.setMaxRetryCount(
+        participantCounter.setMaxRetryCount(
                 clRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().getMaxRetryCount());
-        stateChange.setMaxWaitMs(
-                clRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().getMaxWaitMs());
+        participantCounter
+                .setMaxWaitMs(clRuntimeParameterGroup.getParticipantParameters().getUpdateParameters().getMaxWaitMs());
+
+        maxMessageAgeMs = clRuntimeParameterGroup.getParticipantParameters().getMaxMessageAgeMs();
     }
 
     /**
@@ -117,6 +98,17 @@ public class SupervisionScanner {
     public void run(boolean counterCheck) {
         LOGGER.debug("Scanning control loops in the database . . .");
 
+        if (counterCheck) {
+            try {
+                for (Participant participant : participantProvider.getParticipants(null, null)) {
+                    scanParticipant(participant);
+                }
+            } catch (PfModelException pfme) {
+                LOGGER.warn("error reading participant from database", pfme);
+                return;
+            }
+        }
+
         try {
             for (ControlLoop controlLoop : controlLoopProvider.getControlLoops(null, null)) {
                 scanControlLoop(controlLoop, counterCheck);
@@ -128,6 +120,33 @@ public class SupervisionScanner {
         LOGGER.debug("Control loop scan complete . . .");
     }
 
+    private void scanParticipant(Participant participant) throws PfModelException {
+        ToscaConceptIdentifier id = participant.getKey().asIdentifier();
+        if (participantCounter.isFault(id)) {
+            LOGGER.debug("report Participant fault");
+            return;
+        }
+        if (participantCounter.getDuration(id) > maxMessageAgeMs) {
+            if (participantCounter.count(id)) {
+                LOGGER.debug("retry message ParticipantStatusReq");
+                participantStatusReqPublisher.send(id);
+                participant.setHealthStatus(ParticipantHealthStatus.NOT_HEALTHY);
+            } else {
+                LOGGER.debug("report Participant fault");
+                participantCounter.setFault(id);
+                participant.setHealthStatus(ParticipantHealthStatus.OFF_LINE);
+            }
+            participantProvider.updateParticipants(List.of(participant));
+        }
+    }
+
+    /**
+     * handle participant Status message.
+     */
+    public void handleParticipantStatus(ToscaConceptIdentifier id) {
+        participantCounter.clear(id);
+    }
+
     private void scanControlLoop(final ControlLoop controlLoop, boolean counterCheck) throws PfModelException {
         LOGGER.debug("scanning control loop {} . . .", controlLoop.getKey().asIdentifier());
 
@@ -166,17 +185,17 @@ public class SupervisionScanner {
     }
 
     private void clearFaultAndCounter(ControlLoop controlLoop) {
-        stateChange.clear(controlLoop.getKey().asIdentifier());
+        controlLoopCounter.clear(controlLoop.getKey().asIdentifier());
     }
 
     private void handleCounter(ControlLoop controlLoop) {
         ToscaConceptIdentifier id = controlLoop.getKey().asIdentifier();
-        if (stateChange.isFault(id)) {
+        if (controlLoopCounter.isFault(id)) {
             LOGGER.debug("report ControlLoop fault");
             return;
         }
 
-        if (stateChange.count(id)) {
+        if (controlLoopCounter.count(id)) {
             if (ControlLoopState.UNINITIALISED2PASSIVE.equals(controlLoop.getState())) {
                 LOGGER.debug("retry message ControlLoopUpdate");
                 controlLoopUpdatePublisher.send(controlLoop);
@@ -186,7 +205,7 @@ public class SupervisionScanner {
             }
         } else {
             LOGGER.debug("report ControlLoop fault");
-            stateChange.setFault(id);
+            controlLoopCounter.setFault(id);
         }
     }
 }
diff --git a/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/comm/ParticipantStatusReqPublisher.java b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/comm/ParticipantStatusReqPublisher.java
new file mode 100644
index 000000000..69d598285
--- /dev/null
+++ b/runtime-controlloop/src/main/java/org/onap/policy/clamp/controlloop/runtime/supervision/comm/ParticipantStatusReqPublisher.java
@@ -0,0 +1,48 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.clamp.controlloop.runtime.supervision.comm;
+
+import java.time.Instant;
+import org.onap.policy.clamp.controlloop.models.messages.dmaap.participant.ParticipantStatusReq;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ParticipantStatusReqPublisher extends AbstractParticipantPublisher<ParticipantStatusReq> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ParticipantStatusReqPublisher.class);
+
+    /**
+     * Send ParticipantStatusReq to Participant.
+     *
+     * @param participantId the participant Id
+     */
+    public void send(ToscaConceptIdentifier participantId) {
+        ParticipantStatusReq message = new ParticipantStatusReq();
+        message.setParticipantId(participantId);
+        message.setTimestamp(Instant.now());
+
+        LOGGER.debug("Participant StatusReq sent {}", message);
+        super.send(message);
+    }
+}
diff --git a/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounterTest.java b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounterTest.java
new file mode 100644
index 000000000..51f3b4a32
--- /dev/null
+++ b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/HandleCounterTest.java
@@ -0,0 +1,84 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.clamp.controlloop.runtime.supervision;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class HandleCounterTest {
+
+    private static final int ID = 1;
+
+    @Test
+    void testCount() {
+        var handleCounter = new HandleCounter<Integer>();
+        handleCounter.setMaxRetryCount(2);
+        assertThat(handleCounter.count(ID)).isTrue();
+        assertThat(handleCounter.getCounter(ID)).isEqualTo(1);
+        assertThat(handleCounter.count(ID)).isTrue();
+        assertThat(handleCounter.getCounter(ID)).isEqualTo(2);
+        assertThat(handleCounter.count(ID)).isFalse();
+        assertThat(handleCounter.getCounter(ID)).isEqualTo(2);
+
+        handleCounter.clear(ID);
+        assertThat(handleCounter.count(ID)).isTrue();
+        assertThat(handleCounter.getCounter(ID)).isEqualTo(1);
+    }
+
+    @Test
+    void testFault() {
+        var handleCounter = new HandleCounter<Integer>();
+        handleCounter.setFault(ID);
+        assertThat(handleCounter.isFault(ID)).isTrue();
+        handleCounter.clear(ID);
+        assertThat(handleCounter.isFault(ID)).isFalse();
+    }
+
+    @Test
+    void testDuration() throws InterruptedException {
+
+        var handleCounter = new HandleCounter<Integer>() {
+            long epochMilli = 0;
+
+            @Override
+            protected long getEpochMilli() {
+                return epochMilli;
+            }
+        };
+        handleCounter.epochMilli = 100;
+        var result = handleCounter.getDuration(ID);
+        assertThat(result).isZero();
+
+        handleCounter.epochMilli += 100;
+        result = handleCounter.getDuration(ID);
+        assertThat(result).isEqualTo(100);
+
+        handleCounter.epochMilli += 100;
+        result = handleCounter.getDuration(ID);
+        assertThat(result).isEqualTo(200);
+
+        handleCounter.epochMilli += 100;
+        handleCounter.clear(ID);
+        result = handleCounter.getDuration(ID);
+        assertThat(result).isZero();
+    }
+}
diff --git a/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspectTest.java b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspectTest.java
new file mode 100644
index 000000000..30ee9b1b9
--- /dev/null
+++ b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionAspectTest.java
@@ -0,0 +1,66 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.clamp.controlloop.runtime.supervision;
+
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+
+import org.junit.jupiter.api.Test;
+import org.onap.policy.clamp.controlloop.models.messages.dmaap.participant.ParticipantStatus;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier;
+
+class SupervisionAspectTest {
+
+    @Test
+    void testSchedule() throws Exception {
+        var supervisionScanner = spy(mock(SupervisionScanner.class));
+        try (var supervisionAspect = new SupervisionAspect(supervisionScanner)) {
+            supervisionAspect.schedule();
+            verify(supervisionScanner, timeout(500)).run(eq(true));
+        }
+    }
+
+    @Test
+    void testDoCheck() throws Exception {
+        var supervisionScanner = spy(mock(SupervisionScanner.class));
+        try (var supervisionAspect = new SupervisionAspect(supervisionScanner)) {
+            supervisionAspect.doCheck();
+            supervisionAspect.doCheck();
+            verify(supervisionScanner, timeout(500).times(2)).run(eq(false));
+        }
+    }
+
+    @Test
+    void testHandleParticipantStatus() throws Exception {
+        var supervisionScanner = spy(mock(SupervisionScanner.class));
+        var participantStatusMessage = new ParticipantStatus();
+        var identifier = new ToscaConceptIdentifier("abc", "1.0.0");
+        participantStatusMessage.setParticipantId(identifier);
+
+        try (var supervisionAspect = new SupervisionAspect(supervisionScanner)) {
+            supervisionAspect.handleParticipantStatus(participantStatusMessage);
+            verify(supervisionScanner, timeout(500)).handleParticipantStatus(eq(identifier));
+        }
+    }
+}
diff --git a/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScannerTest.java b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScannerTest.java
new file mode 100644
index 000000000..485f58dba
--- /dev/null
+++ b/runtime-controlloop/src/test/java/org/onap/policy/clamp/controlloop/runtime/supervision/SupervisionScannerTest.java
@@ -0,0 +1,160 @@
+/*-
+ * ============LICENSE_START=======================================================
+ *  Copyright (C) 2021 Nordix Foundation.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.clamp.controlloop.runtime.supervision;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.junit.jupiter.api.Test;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoop;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoopElement;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoopOrderedState;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ControlLoopState;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.Participant;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantHealthStatus;
+import org.onap.policy.clamp.controlloop.models.controlloop.concepts.ParticipantState;
+import org.onap.policy.clamp.controlloop.models.controlloop.persistence.provider.ControlLoopProvider;
+import org.onap.policy.clamp.controlloop.models.controlloop.persistence.provider.ParticipantProvider;
+import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ControlLoopStateChangePublisher;
+import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ControlLoopUpdatePublisher;
+import org.onap.policy.clamp.controlloop.runtime.supervision.comm.ParticipantStatusReqPublisher;
+import org.onap.policy.clamp.controlloop.runtime.util.CommonTestData;
+import org.onap.policy.models.base.PfModelException;
+import org.onap.policy.models.tosca.authorative.concepts.ToscaConceptIdentifier;
+
+class SupervisionScannerTest {
+
+    @Test
+    void testScannerOrderedStateEqualsToState() throws PfModelException {
+        var controlLoopProvider = mock(ControlLoopProvider.class);
+        var controlLoopStateChangePublisher = mock(ControlLoopStateChangePublisher.class);
+        var controlLoopUpdatePublisher = mock(ControlLoopUpdatePublisher.class);
+        var participantProvider = mock(ParticipantProvider.class);
+        var participantStatusReqPublisher = mock(ParticipantStatusReqPublisher.class);
+        var clRuntimeParameterGroup = CommonTestData.geParameterGroup("dbScanner");
+
+        var controlLoop = new ControlLoop();
+        when(controlLoopProvider.getControlLoops(null, null)).thenReturn(List.of(controlLoop));
+
+        var supervisionScanner =
+                new SupervisionScanner(controlLoopProvider, controlLoopStateChangePublisher, controlLoopUpdatePublisher,
+                        participantProvider, participantStatusReqPublisher, clRuntimeParameterGroup);
+        supervisionScanner.run(false);
+
+        verify(controlLoopProvider, times(0)).updateControlLoop(any(ControlLoop.class));
+    }
+
+    @Test
+    void testScannerOrderedStateDifferentToState() throws PfModelException {
+        var controlLoop = new ControlLoop();
+        controlLoop.setState(ControlLoopState.UNINITIALISED2PASSIVE);
+        controlLoop.setOrderedState(ControlLoopOrderedState.UNINITIALISED);
+        controlLoop.setElements(Map.of(UUID.randomUUID(), new ControlLoopElement()));
+        var controlLoopProvider = mock(ControlLoopProvider.class);
+        when(controlLoopProvider.getControlLoops(null, null)).thenReturn(List.of(controlLoop));
+
+        var controlLoopUpdatePublisher = mock(ControlLoopUpdatePublisher.class);
+        var controlLoopStateChangePublisher = mock(ControlLoopStateChangePublisher.class);
+        var participantProvider = mock(ParticipantProvider.class);
+        var participantStatusReqPublisher = mock(ParticipantStatusReqPublisher.class);
+        var clRuntimeParameterGroup = CommonTestData.geParameterGroup("dbScanner");
+
+        var supervisionScanner =
+                new SupervisionScanner(controlLoopProvider, controlLoopStateChangePublisher, controlLoopUpdatePublisher,
+                        participantProvider, participantStatusReqPublisher, clRuntimeParameterGroup);
+        supervisionScanner.run(false);
+
+        verify(controlLoopProvider, times(1)).updateControlLoop(any(ControlLoop.class));
+    }
+
+    @Test
+    void testScanner() throws PfModelException {
+        var controlLoopProvider = mock(ControlLoopProvider.class);
+        var controlLoop = new ControlLoop();
+        when(controlLoopProvider.getControlLoops(null, null)).thenReturn(List.of(controlLoop));
+
+        var participantProvider = mock(ParticipantProvider.class);
+        var participant = new Participant();
+        participant.setName("Participant0");
+        participant.setVersion("1.0.0");
+        when(participantProvider.getParticipants(null, null)).thenReturn(List.of(participant));
+
+        var controlLoopUpdatePublisher = mock(ControlLoopUpdatePublisher.class);
+        var participantStatusReqPublisher = mock(ParticipantStatusReqPublisher.class);
+        var controlLoopStateChangePublisher = mock(ControlLoopStateChangePublisher.class);
+        var clRuntimeParameterGroup = CommonTestData.geParameterGroup("dbScanner");
+
+        var supervisionScanner =
+                new SupervisionScanner(controlLoopProvider, controlLoopStateChangePublisher, controlLoopUpdatePublisher,
+                        participantProvider, participantStatusReqPublisher, clRuntimeParameterGroup);
+
+        supervisionScanner.handleParticipantStatus(participant.getKey().asIdentifier());
+        supervisionScanner.run(true);
+        verify(controlLoopProvider, times(0)).updateControlLoop(any(ControlLoop.class));
+        verify(participantStatusReqPublisher, times(0)).send(any(ToscaConceptIdentifier.class));
+    }
+
+    @Test
+    void testScanParticipant() throws PfModelException {
+        var controlLoopProvider = mock(ControlLoopProvider.class);
+        var controlLoop = new ControlLoop();
+        when(controlLoopProvider.getControlLoops(null, null)).thenReturn(List.of(controlLoop));
+
+        var clRuntimeParameterGroup = CommonTestData.geParameterGroup("dbScanParticipant");
+        clRuntimeParameterGroup.getParticipantParameters().setMaxMessageAgeMs(0);
+
+        var participant = new Participant();
+        participant.setName("Participant0");
+        participant.setVersion("1.0.0");
+        participant.setHealthStatus(ParticipantHealthStatus.HEALTHY);
+        participant.setParticipantState(ParticipantState.ACTIVE);
+        participant.setDefinition(new ToscaConceptIdentifier("unknown", "0.0.0"));
+        var participantProvider = new ParticipantProvider(clRuntimeParameterGroup.getDatabaseProviderParameters());
+        participantProvider.updateParticipants(List.of(participant));
+
+        var controlLoopUpdatePublisher = mock(ControlLoopUpdatePublisher.class);
+        var participantStatusReqPublisher = mock(ParticipantStatusReqPublisher.class);
+        var controlLoopStateChangePublisher = mock(ControlLoopStateChangePublisher.class);
+
+        var supervisionScanner =
+                new SupervisionScanner(controlLoopProvider, controlLoopStateChangePublisher, controlLoopUpdatePublisher,
+                        participantProvider, participantStatusReqPublisher, clRuntimeParameterGroup);
+
+        supervisionScanner.handleParticipantStatus(participant.getKey().asIdentifier());
+        supervisionScanner.run(true);
+        verify(participantStatusReqPublisher, times(1)).send(any(ToscaConceptIdentifier.class));
+
+        List<Participant> participants = participantProvider.getParticipants(null, null);
+        assertThat(participants.get(0).getHealthStatus()).isEqualTo(ParticipantHealthStatus.NOT_HEALTHY);
+
+        supervisionScanner.run(true);
+        supervisionScanner.run(true);
+        participants = participantProvider.getParticipants(null, null);
+        assertThat(participants.get(0).getHealthStatus()).isEqualTo(ParticipantHealthStatus.OFF_LINE);
+    }
+}