From 58eae5d260f4e41ae439b733404fd1bc7dca453b Mon Sep 17 00:00:00 2001
From: Rashmi Pujar <rashmi.pujar1@bell.ca>
Date: Fri, 7 Jan 2022 01:38:40 -0500
Subject: Prometheues metrics for APEX engine stats

Issue-ID: POLICY-3846
Signed-off-by: Rashmi Pujar <rashmi.pujar1@bell.ca>
Change-Id: I385446683372548f3fed6d609a9e85633ec2916d
---
 .../core/engine/engine/impl/ApexEngineImpl.java    | 24 ++++++-
 .../engine/engine/impl/ApexEngineImplTest.java     | 67 ++++++++++++++++++--
 model/engine-model/pom.xml                         |  7 +-
 .../model/enginemodel/concepts/AxEngineState.java  | 31 ++++++---
 .../model/enginemodel/concepts/AxEngineStats.java  | 54 +++++++++++++++-
 .../enginemodel/concepts/EngineStatsTest.java      | 74 +++++++++++++++++++++-
 6 files changed, 234 insertions(+), 23 deletions(-)

diff --git a/core/core-engine/src/main/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImpl.java b/core/core-engine/src/main/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImpl.java
index 0d9d34d39..7bd9087df 100644
--- a/core/core-engine/src/main/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImpl.java
+++ b/core/core-engine/src/main/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImpl.java
@@ -2,7 +2,7 @@
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2016-2018 Ericsson. All rights reserved.
  *  Modifications Copyright (C) 2019-2020 Nordix Foundation.
- *  Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
+ *  Modifications Copyright (C) 2021-2022 Bell Canada. All rights reserved.
  *  Modifications Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,6 +25,7 @@ package org.onap.policy.apex.core.engine.engine.impl;
 
 import static org.onap.policy.common.utils.validation.Assertions.argumentNotNull;
 
+import io.prometheus.client.Gauge;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -72,6 +73,11 @@ public class ApexEngineImpl implements ApexEngine {
     // Logger for this class
     private static final XLogger LOGGER = XLoggerFactory.getXLogger(ApexEngineImpl.class);
 
+    // Register state changes with prometheus
+    static final Gauge ENGINE_STATE = Gauge.build().name("apex_engine_state").labelNames("engine_instance_id")
+            .help("State of the APEX engine as integers mapped as - 0:UNDEFINED, 1:STOPPED, 2:READY,"
+                    + " 3:EXECUTING, 4:STOPPING").register();
+
     // Recurring string constants
     private static final String UPDATE_MODEL = "updateModel()<-";
     private static final String START = "start()<-";
@@ -123,6 +129,7 @@ public class ApexEngineImpl implements ApexEngine {
      */
     @Override
     public void updateModel(final AxPolicyModel apexModel, final boolean isSubsequentInstance) throws ApexException {
+        updateStatePrometheusMetric();
         if (apexModel != null) {
             LOGGER.entry("updateModel()->{}, apexPolicyModel {}", key.getId(), apexModel.getKey().getId());
         } else {
@@ -249,6 +256,7 @@ public class ApexEngineImpl implements ApexEngine {
 
         // OK, we are good to go
         state = AxEngineState.READY;
+        updateStatePrometheusMetric();
 
         LOGGER.exit("start()" + key);
     }
@@ -278,6 +286,7 @@ public class ApexEngineImpl implements ApexEngine {
                     case READY:
                     case STOPPED:
                         state = AxEngineState.STOPPED;
+                        updateStatePrometheusMetric();
                         stateMachineHandler.stop();
                         engineStats.engineStop();
                         LOGGER.exit("stop()" + key);
@@ -286,6 +295,7 @@ public class ApexEngineImpl implements ApexEngine {
                     // Engine is executing a policy, wait for it to stop
                     case EXECUTING:
                         state = AxEngineState.STOPPING;
+                        updateStatePrometheusMetric();
                         break;
 
                     // Wait for the engine to stop
@@ -303,6 +313,7 @@ public class ApexEngineImpl implements ApexEngine {
         synchronized (stateLockObj) {
             state = AxEngineState.STOPPED;
         }
+        updateStatePrometheusMetric();
 
         throw new ApexException(STOP + key.getId() + "," + state + ", error stopping engine, engine stop timed out");
     }
@@ -370,6 +381,7 @@ public class ApexEngineImpl implements ApexEngine {
 
             state = AxEngineState.EXECUTING;
         }
+        updateStatePrometheusMetric();
 
         String message = "execute(): triggered by event " + incomingEvent.toString();
         LOGGER.debug(message);
@@ -413,6 +425,7 @@ public class ApexEngineImpl implements ApexEngine {
                 state = AxEngineState.STOPPED;
             }
         }
+        updateStatePrometheusMetric();
         return ret;
     }
 
@@ -507,4 +520,11 @@ public class ApexEngineImpl implements ApexEngine {
 
         return Set.of(exceptionEvent);
     }
-}
+
+    /**
+     * Update the APEX engine state to prometheus for monitoring.
+     */
+    private void updateStatePrometheusMetric() {
+        ENGINE_STATE.labels(getKey().getId()).set(state.getStateIdentifier());
+    }
+}
\ No newline at end of file
diff --git a/core/core-engine/src/test/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImplTest.java b/core/core-engine/src/test/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImplTest.java
index c9d9b9c8d..88602206d 100644
--- a/core/core-engine/src/test/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImplTest.java
+++ b/core/core-engine/src/test/java/org/onap/policy/apex/core/engine/engine/impl/ApexEngineImplTest.java
@@ -30,6 +30,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+import io.prometheus.client.CollectorRegistry;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.util.HashMap;
@@ -70,6 +71,8 @@ import org.onap.policy.common.parameters.ParameterService;
  * Test the engine implementation.
  */
 public class ApexEngineImplTest {
+    private static final String ENGINE_ID = "Engine:0.0.1";
+
     private AxPolicyModel policyModel;
     private AxPolicyModel incompatiblePolicyModel;
     private AxPolicyModel policyModelWithStates;
@@ -170,7 +173,7 @@ public class ApexEngineImplTest {
 
     @Test
     public void testSanity() throws ApexException  {
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
         assertNotNull(engine);
         assertEquals(engineKey, engine.getKey());
@@ -182,6 +185,7 @@ public class ApexEngineImplTest {
             .hasMessage("stop()<-Engine:0.0.1,STOPPED, cannot stop engine, " + "engine is already stopped");
 
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
         assertEquals(0, engine.getEngineContext().size());
         assertEquals(engineKey, engine.getEngineStatus().getKey());
         assertNull(engine.getInternalContext());
@@ -200,7 +204,7 @@ public class ApexEngineImplTest {
 
     @Test
     public void testListener() throws ApexException {
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
 
         engine.addEventListener("myListener", new DummyListener());
@@ -227,6 +231,7 @@ public class ApexEngineImplTest {
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertThatThrownBy(engine::start)
             .hasMessage("start()<-Engine:0.0.1,READY, cannot start engine, engine not in state STOPPED");
@@ -236,25 +241,29 @@ public class ApexEngineImplTest {
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.clear();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         assertThatThrownBy(engine::start).hasMessage("start()<-Engine:0.0.1,STOPPED,  cannot start engine, "
             + "engine has not been initialized, its model is not loaded");
 
         engine.updateModel(policyModel, false);
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertNull(engine.createEvent(null));
     }
 
     @Test
     public void testEventKey() throws ApexException {
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
         engine.updateModel(policyModel, false);
         engine.start();
@@ -265,33 +274,41 @@ public class ApexEngineImplTest {
 
         assertTrue(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.addEventListener("myListener", new DummyListener());
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertThatThrownBy(() -> engine.updateModel(policyModel, false)).hasMessage(
             "updateModel()<-Engine:0.0.1, cannot update model, engine should be stopped but is in state READY");
 
         assertTrue(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.addEventListener("badListener", new DummyEnEventListener());
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertFalse(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.removeEventListener("badListener");
         engine.addEventListener("slowListener", new DummySlowEnEventListener());
@@ -299,21 +316,24 @@ public class ApexEngineImplTest {
 
     @Test
     public void testState() throws InterruptedException, ApexException {
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
         assertNotNull(engine);
         assertEquals(engineKey, engine.getKey());
 
         engine.updateModel(policyModel, false);
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         DummySlowEnEventListener slowListener = new DummySlowEnEventListener();
         engine.addEventListener("slowListener", slowListener);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         AxArtifactKey eventKey = new AxArtifactKey("Event:0.0.1");
         EnEvent event = engine.createEvent(eventKey);
@@ -329,15 +349,18 @@ public class ApexEngineImplTest {
         }).start();
         await().atLeast(50, TimeUnit.MILLISECONDS).until(() -> engine.getState().equals(AxEngineState.EXECUTING));
         assertEquals(AxEngineState.EXECUTING, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.EXECUTING);
 
         assertFalse(engine.handleEvent(event));
         assertNotNull(engine.createEvent(eventKey));
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         // 4 seconds is more than the 3 second wait on engine stopping
         slowListener.setWaitTime(4000);
@@ -350,24 +373,28 @@ public class ApexEngineImplTest {
 
         await().atLeast(50, TimeUnit.MILLISECONDS).until(() -> engine.getState().equals(AxEngineState.EXECUTING));
         assertEquals(AxEngineState.EXECUTING, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.EXECUTING);
         assertThatThrownBy(engine::stop)
             .hasMessage("stop()<-Engine:0.0.1,STOPPED, error stopping engine, engine stop timed out");
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
         engine.clear();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
     }
 
     @Test
     public void testStateMachineError() throws InterruptedException, IllegalArgumentException, IllegalAccessException,
                     NoSuchFieldException, SecurityException, ApexException {
 
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
         assertNotNull(engine);
         assertEquals(engineKey, engine.getKey());
 
         engine.updateModel(policyModel, false);
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         final Field smHandlerField = engine.getClass().getDeclaredField("stateMachineHandler");
         smHandlerField.setAccessible(true);
@@ -375,8 +402,10 @@ public class ApexEngineImplTest {
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         AxArtifactKey eventKey = new AxArtifactKey("Event:0.0.1");
         EnEvent event = engine.createEvent(eventKey);
@@ -384,32 +413,38 @@ public class ApexEngineImplTest {
 
         assertFalse(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
         Mockito.doThrow(new StateMachineException("mocked state machine exception",
                 new IOException("nexted exception"))).when(smHandlerMock).start();
         assertThatThrownBy(engine::start).hasMessage("updateModel()<-Engine:0.0.1, error starting the engine state "
                 + "machines \"Engine:0.0.1\"");
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.clear();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
     }
 
     @Test
     public void testStateMachineHandler() throws InterruptedException, IllegalArgumentException, IllegalAccessException,
                     NoSuchFieldException, SecurityException, ApexException {
-        AxArtifactKey engineKey = new AxArtifactKey("Engine:0.0.1");
+        AxArtifactKey engineKey = new AxArtifactKey(ENGINE_ID);
         ApexEngineImpl engine = (ApexEngineImpl) new ApexEngineFactory().createApexEngine(engineKey);
         assertNotNull(engine);
         assertEquals(engineKey, engine.getKey());
 
         engine.updateModel(policyModelWithStates, false);
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         AxArtifactKey eventKey = new AxArtifactKey("Event:0.0.1");
         EnEvent event = engine.createEvent(eventKey);
@@ -417,17 +452,22 @@ public class ApexEngineImplTest {
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         // Can't work, state is not fully defined
         assertFalse(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         final Field smHandlerField = engine.getClass().getDeclaredField("stateMachineHandler");
         smHandlerField.setAccessible(true);
@@ -448,22 +488,35 @@ public class ApexEngineImplTest {
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         assertThatThrownBy(engine::start).hasMessageContaining("updateModel()<-Engine:0.0.1, error starting the "
                     + "engine state machines \"Engine:0.0.1\"");
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.start();
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         // Works, Dummy executor fakes event execution
         assertTrue(engine.handleEvent(event));
         assertEquals(AxEngineState.READY, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.READY);
 
         engine.stop();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
 
         engine.clear();
         assertEquals(AxEngineState.STOPPED, engine.getState());
+        checkAxEngineStateMetric(AxEngineState.STOPPED);
+    }
+
+    private void checkAxEngineStateMetric(AxEngineState state) {
+        Double stateMetric = CollectorRegistry.defaultRegistry
+                .getSampleValue("apex_engine_state", new String[]{"engine_instance_id"},
+                        new String[]{ENGINE_ID});
+        assertEquals(stateMetric.intValue(), state.getStateIdentifier());
     }
-}
+}
\ No newline at end of file
diff --git a/model/engine-model/pom.xml b/model/engine-model/pom.xml
index a2a247d43..7a1b485f8 100644
--- a/model/engine-model/pom.xml
+++ b/model/engine-model/pom.xml
@@ -1,6 +1,7 @@
 <!--
   ============LICENSE_START=======================================================
    Copyright (C) 2018 Ericsson. All rights reserved.
+   Modifications Copyright (C) 2022 Bell Canada.
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -35,6 +36,10 @@
             <artifactId>policy-model</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -62,4 +67,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>
\ No newline at end of file
diff --git a/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineState.java b/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineState.java
index f77fe1910..63c8706d2 100644
--- a/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineState.java
+++ b/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineState.java
@@ -1,19 +1,20 @@
 /*-
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2016-2018 Ericsson. All rights reserved.
+ *  Modifications Copyright (C) 2022 Bell Canada. All rights reserved.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- * 
+ *
  * SPDX-License-Identifier: Apache-2.0
  * ============LICENSE_END=========================================================
  */
@@ -34,13 +35,23 @@ import javax.xml.bind.annotation.XmlType;
 @XmlType(name = "AxEngineState", namespace = "http://www.onap.org/policy/apex-pdp")
 public enum AxEngineState {
     /** The state of the engine is not known. */
-    UNDEFINED,
+    UNDEFINED(0),
     /** The engine is stopped. */
-    STOPPED,
+    STOPPED(1),
     /** The engine is running and is waiting to execute a policy. */
-    READY,
+    READY(2),
     /** The engine is running and is executing a policy. */
-    EXECUTING,
-    /** The engine has been ordered to stop and is stoping. */
-    STOPPING;
-}
+    EXECUTING(3),
+    /** The engine has been ordered to stop and is stopping. */
+    STOPPING(4);
+
+    private final int stateIdentifier;
+
+    AxEngineState(int stateIdentifier) {
+        this.stateIdentifier = stateIdentifier;
+    }
+
+    public int getStateIdentifier() {
+        return stateIdentifier;
+    }
+}
\ No newline at end of file
diff --git a/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineStats.java b/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineStats.java
index 916525900..39a90800d 100644
--- a/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineStats.java
+++ b/model/engine-model/src/main/java/org/onap/policy/apex/model/enginemodel/concepts/AxEngineStats.java
@@ -2,6 +2,7 @@
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2016-2018 Ericsson. All rights reserved.
  *  Modifications Copyright (C) 2019-2020 Nordix Foundation.
+ *  Modifications Copyright (C) 2022 Bell Canada.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,6 +22,8 @@
 
 package org.onap.policy.apex.model.enginemodel.concepts;
 
+import io.prometheus.client.Gauge;
+import io.prometheus.client.Histogram;
 import java.text.SimpleDateFormat;
 import java.util.List;
 import javax.persistence.Column;
@@ -59,6 +62,22 @@ import org.onap.policy.common.utils.validation.Assertions;
 public class AxEngineStats extends AxConcept {
     private static final long serialVersionUID = -6981129081962785368L;
     private static final int HASH_CODE_PRIME = 32;
+    static final String ENGINE_INSTANCE_ID = "engine_instance_id";
+    static final Gauge ENGINE_EVENTS_EXECUTED_COUNT = Gauge.build().name("apex_engine_events_executed_count")
+            .labelNames(ENGINE_INSTANCE_ID)
+            .help("Total number of APEX events processed by the engine.").register();
+    static final Gauge ENGINE_UPTIME = Gauge.build().name("apex_engine_uptime")
+            .labelNames(ENGINE_INSTANCE_ID)
+            .help("Time elapsed since the engine was started.").register();
+    static final Gauge ENGINE_START_TIMESTAMP = Gauge.build().name("apex_engine_last_start_timestamp_epoch")
+            .labelNames(ENGINE_INSTANCE_ID)
+            .help("Epoch timestamp of the instance when engine was last started.").register();
+    static final Gauge ENGINE_AVG_EXECUTION_TIME = Gauge.build().name("apex_engine_average_execution_time_seconds")
+            .labelNames(ENGINE_INSTANCE_ID)
+            .help("Average time taken to execute an APEX policy in seconds.").register();
+    static final Histogram ENGINE_LAST_EXECUTION_TIME = Histogram.build()
+            .name("apex_engine_last_execution_time").labelNames(ENGINE_INSTANCE_ID)
+            .help("Time taken to execute the last APEX policy in seconds.").register();
 
     @EmbeddedId
     @XmlElement(name = "key", required = true)
@@ -104,6 +123,22 @@ public class AxEngineStats extends AxConcept {
         upTime = 0;
         lastEnterTime = 0;
         lastStart = 0;
+        initEngineMetricsWithPrometheus();
+    }
+
+    /**
+     * Register the APEX engine metrics with Prometheus.
+     */
+    private void initEngineMetricsWithPrometheus() {
+        var engineId = getKey().getParentArtifactKey().getId();
+        if (engineId.startsWith(AxKey.NULL_KEY_NAME)) {
+            return;
+        }
+        ENGINE_UPTIME.labels(engineId).set(upTime / 1000d);
+        ENGINE_EVENTS_EXECUTED_COUNT.labels(engineId).set(this.eventCount);
+        ENGINE_START_TIMESTAMP.labels(engineId).set(this.lastStart);
+        ENGINE_AVG_EXECUTION_TIME.labels(engineId).set(this.averageExecutionTime / 1000d);
+        ENGINE_LAST_EXECUTION_TIME.labels(engineId).observe(this.lastExecutionTime / 1000d);
     }
 
     /**
@@ -147,6 +182,7 @@ public class AxEngineStats extends AxConcept {
         this.averageExecutionTime = averageExecutionTime;
         this.upTime = upTime;
         this.lastStart = lastStart;
+        initEngineMetricsWithPrometheus();
     }
 
     /**
@@ -218,6 +254,8 @@ public class AxEngineStats extends AxConcept {
      */
     public void setEventCount(final long eventCount) {
         this.eventCount = eventCount;
+        ENGINE_EVENTS_EXECUTED_COUNT.labels(getKey().getParentArtifactKey().getId())
+                .set(this.eventCount);
     }
 
     /**
@@ -236,6 +274,8 @@ public class AxEngineStats extends AxConcept {
      */
     public void setLastExecutionTime(final long lastExecutionTime) {
         this.lastExecutionTime = lastExecutionTime;
+        ENGINE_LAST_EXECUTION_TIME.labels(getKey().getParentArtifactKey().getId())
+                .observe(this.lastExecutionTime / 1000d);
     }
 
     /**
@@ -254,6 +294,8 @@ public class AxEngineStats extends AxConcept {
      */
     public void setAverageExecutionTime(final double averageExecutionTime) {
         this.averageExecutionTime = averageExecutionTime;
+        ENGINE_AVG_EXECUTION_TIME.labels(getKey().getParentArtifactKey().getId())
+                .set(this.averageExecutionTime / 1000d);
     }
 
     /**
@@ -275,6 +317,7 @@ public class AxEngineStats extends AxConcept {
      */
     public void setUpTime(final long upTime) {
         this.upTime = upTime;
+        ENGINE_UPTIME.labels(getKey().getParentArtifactKey().getId()).set(this.upTime / 1000d);
     }
 
     /**
@@ -284,6 +327,7 @@ public class AxEngineStats extends AxConcept {
      */
     private void setLastStart(final long lastStart) {
         this.lastStart = lastStart;
+        ENGINE_START_TIMESTAMP.labels(getKey().getParentArtifactKey().getId()).set(this.lastStart);
     }
 
     /**
@@ -306,6 +350,7 @@ public class AxEngineStats extends AxConcept {
         upTime = 0;
         lastEnterTime = 0;
         lastStart = 0;
+        initEngineMetricsWithPrometheus();
     }
 
     /**
@@ -321,6 +366,7 @@ public class AxEngineStats extends AxConcept {
         }
         lastEnterTime = now;
         timeStamp = now;
+        ENGINE_EVENTS_EXECUTED_COUNT.labels(getKey().getParentArtifactKey().getId()).set(this.eventCount);
     }
 
     /**
@@ -329,10 +375,14 @@ public class AxEngineStats extends AxConcept {
     public synchronized void executionExit() {
         final long now = System.currentTimeMillis();
         lastExecutionTime = now - lastEnterTime;
+        ENGINE_LAST_EXECUTION_TIME.labels(getKey().getParentArtifactKey().getId())
+                .observe(this.lastExecutionTime / 1000d);
 
         averageExecutionTime = ((averageExecutionTime * (eventCount - 1.0)) + lastExecutionTime) / eventCount;
         lastEnterTime = 0;
         timeStamp = System.currentTimeMillis();
+        ENGINE_AVG_EXECUTION_TIME.labels(getKey().getParentArtifactKey().getId())
+                .set(this.averageExecutionTime / 1000d);
     }
 
     /**
@@ -352,6 +402,7 @@ public class AxEngineStats extends AxConcept {
         timeStamp = now;
         upTime += (timeStamp - this.getLastStart());
         this.setLastStart(0);
+        ENGINE_UPTIME.labels(getKey().getParentArtifactKey().getId()).set(this.upTime / 1000d);
     }
 
     /**
@@ -417,6 +468,7 @@ public class AxEngineStats extends AxConcept {
         copy.setAverageExecutionTime(averageExecutionTime);
         copy.setUpTime(upTime);
         copy.setLastStart(lastStart);
+        initEngineMetricsWithPrometheus();
 
         return copy;
     }
@@ -514,4 +566,4 @@ public class AxEngineStats extends AxConcept {
 
         return Long.compare(lastStart, other.lastStart);
     }
-}
+}
\ No newline at end of file
diff --git a/model/engine-model/src/test/java/org/onap/policy/apex/model/enginemodel/concepts/EngineStatsTest.java b/model/engine-model/src/test/java/org/onap/policy/apex/model/enginemodel/concepts/EngineStatsTest.java
index 7669f67bd..9ecfcdd2c 100644
--- a/model/engine-model/src/test/java/org/onap/policy/apex/model/enginemodel/concepts/EngineStatsTest.java
+++ b/model/engine-model/src/test/java/org/onap/policy/apex/model/enginemodel/concepts/EngineStatsTest.java
@@ -2,6 +2,7 @@
  * ============LICENSE_START=======================================================
  *  Copyright (C) 2016-2018 Ericsson. All rights reserved.
  *  Modifications Copyright (C) 2019-2020 Nordix Foundation.
+ *  Modifications Copyright (C) 2022 Bell Canada.
  * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +29,7 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import io.prometheus.client.CollectorRegistry;
 import org.junit.Test;
 import org.onap.policy.apex.model.basicmodel.concepts.AxArtifactKey;
 import org.onap.policy.apex.model.basicmodel.concepts.AxReferenceKey;
@@ -41,13 +43,15 @@ import org.onap.policy.apex.model.basicmodel.concepts.AxValidationResult.Validat
  */
 public class EngineStatsTest {
     private static final Object WAIT_LOCK = new Object();
+    private static final String ENGINE_KEY = "EngineKey";
+    private static final String ENGINE_VERSION = "0.0.1";
 
     @Test
     public void testEngineStats() {
         assertNotNull(new AxEngineStats());
         assertNotNull(new AxEngineStats(new AxReferenceKey()));
 
-        final AxReferenceKey statsKey = new AxReferenceKey("EngineKey", "0.0.1", "EngineStats");
+        final AxReferenceKey statsKey = new AxReferenceKey(ENGINE_KEY, ENGINE_VERSION, "EngineStats");
         final AxEngineStats stats = new AxEngineStats(statsKey);
 
         assertThatThrownBy(() -> stats.setKey(null))
@@ -58,13 +62,16 @@ public class EngineStatsTest {
 
         stats.setAverageExecutionTime(123.45);
         assertEquals(Double.valueOf(123.45), Double.valueOf(stats.getAverageExecutionTime()));
+        checkAvgExecTimeMetric(stats);
 
         stats.setEventCount(987);
         assertEquals(987, stats.getEventCount());
+        checkEventsCountMetric(stats);
 
         final long lastExecutionTime = System.currentTimeMillis();
         stats.setLastExecutionTime(lastExecutionTime);
         assertEquals(lastExecutionTime, stats.getLastExecutionTime());
+        checkLastExecTimeMetric(stats);
 
         final long timestamp = System.currentTimeMillis();
         stats.setTimeStamp(timestamp);
@@ -74,28 +81,39 @@ public class EngineStatsTest {
         final long upTime = System.currentTimeMillis() - timestamp;
         stats.setUpTime(upTime);
         assertEquals(upTime, stats.getUpTime());
+        checkUpTimeMetric(stats);
 
         stats.engineStart();
         assertTrue(stats.getUpTime() > -1);
+        checkEngineStartTimestampMetric(stats);
+        checkLastExecTimeMetric(stats);
         stats.engineStop();
         assertTrue(stats.getUpTime() >= 0);
 
         stats.engineStop();
+        checkUpTimeMetric(stats);
+        checkEngineStartTimestampMetric(stats);
 
         stats.reset();
 
         stats.setEventCount(-2);
         stats.executionEnter(new AxArtifactKey());
         assertEquals(2, stats.getEventCount());
+        checkEventsCountMetric(stats);
 
         stats.setEventCount(10);
         stats.executionEnter(new AxArtifactKey());
         assertEquals(11, stats.getEventCount());
+        checkEventsCountMetric(stats);
 
         stats.reset();
         stats.engineStart();
         stats.setEventCount(4);
         stats.executionEnter(new AxArtifactKey());
+        checkEventsCountMetric(stats);
+        checkUpTimeMetric(stats);
+        checkAvgExecTimeMetric(stats);
+        checkEngineStartTimestampMetric(stats);
 
         synchronized (WAIT_LOCK) {
             try {
@@ -109,6 +127,7 @@ public class EngineStatsTest {
         final double avExecutionTime = stats.getAverageExecutionTime();
         assertTrue(avExecutionTime >= 2.0 && avExecutionTime < 10.0);
         stats.engineStop();
+        checkUpTimeMetric(stats);
 
         AxValidationResult result = new AxValidationResult();
         result = stats.validate(result);
@@ -126,6 +145,7 @@ public class EngineStatsTest {
 
         stats.clean();
         stats.reset();
+        checkAllPrometheusMetrics(stats);
 
         final AxEngineStats clonedStats = new AxEngineStats(stats);
         assertEquals("AxEngineStats:(engineKey=AxReferenceKey:(parentKey", clonedStats.toString().substring(0, 50));
@@ -138,6 +158,7 @@ public class EngineStatsTest {
         assertEquals(stats, stats); // NOSONAR
         assertEquals(stats, clonedStats);
         assertNotNull(stats);
+        checkAllPrometheusMetrics(clonedStats);
 
         Object helloObject = "Hello";
         assertNotEquals(stats, helloObject);
@@ -155,6 +176,7 @@ public class EngineStatsTest {
         stats.setTimeStamp(0);
         assertEquals(stats, new AxEngineStats(statsKey));
         assertEquals(0, stats.compareTo(new AxEngineStats(statsKey)));
+        checkAllPrometheusMetrics(clonedStats);
 
         stats.setEventCount(1);
         assertNotEquals(stats, new AxEngineStats(statsKey));
@@ -193,9 +215,57 @@ public class EngineStatsTest {
         assertNotEquals(stats, newStats);
         assertNotEquals(0, stats.compareTo(newStats));
         stats.engineStop();
+        checkUpTimeMetric(stats);
+        checkEngineStartTimestampMetric(stats);
         stats.reset();
         assertEquals(stats, new AxEngineStats(statsKey));
         assertEquals(0, stats.compareTo(new AxEngineStats(statsKey)));
+        checkAllPrometheusMetrics(stats);
     }
 
-}
+    private void checkUpTimeMetric(AxEngineStats stats) {
+        Double upTimeMetric = CollectorRegistry.defaultRegistry.getSampleValue("apex_engine_uptime",
+                new String[]{AxEngineStats.ENGINE_INSTANCE_ID},
+                new String[]{ENGINE_KEY + ":" + ENGINE_VERSION}) * 1000d;
+        assertEquals(upTimeMetric.longValue(), stats.getUpTime());
+    }
+
+    private void checkEventsCountMetric(AxEngineStats stats) {
+        Double eventsCountMetric = CollectorRegistry.defaultRegistry
+                .getSampleValue("apex_engine_events_executed_count",
+                        new String[]{AxEngineStats.ENGINE_INSTANCE_ID},
+                        new String[]{ENGINE_KEY + ":" + ENGINE_VERSION});
+        assertEquals(eventsCountMetric.longValue(), stats.getEventCount());
+    }
+
+    private void checkLastExecTimeMetric(AxEngineStats stats) {
+        Double lastExecTimeMetric = CollectorRegistry.defaultRegistry
+                .getSampleValue("apex_engine_last_execution_time_sum", new String[]{AxEngineStats.ENGINE_INSTANCE_ID},
+                        new String[]{ENGINE_KEY + ":" + ENGINE_VERSION}) * 1000d;
+        assertEquals(lastExecTimeMetric.longValue(), stats.getLastExecutionTime());
+    }
+
+    private void checkEngineStartTimestampMetric(AxEngineStats stats) {
+        Double engineStartTimestampMetric = CollectorRegistry.defaultRegistry
+                .getSampleValue("apex_engine_last_start_timestamp_epoch",
+                        new String[]{AxEngineStats.ENGINE_INSTANCE_ID},
+                        new String[]{ENGINE_KEY + ":" + ENGINE_VERSION});
+        assertEquals(engineStartTimestampMetric.longValue(), stats.getLastStart());
+    }
+
+    private void checkAvgExecTimeMetric(AxEngineStats stats) {
+        Double avgExecTimeMetric = CollectorRegistry.defaultRegistry
+                .getSampleValue("apex_engine_average_execution_time_seconds",
+                        new String[]{AxEngineStats.ENGINE_INSTANCE_ID},
+                        new String[]{ENGINE_KEY + ":" + ENGINE_VERSION}) * 1000d;
+        assertEquals(avgExecTimeMetric, Double.valueOf(stats.getAverageExecutionTime()));
+    }
+
+    private void checkAllPrometheusMetrics(AxEngineStats stats) {
+        checkEventsCountMetric(stats);
+        checkUpTimeMetric(stats);
+        checkAvgExecTimeMetric(stats);
+        checkEngineStartTimestampMetric(stats);
+        checkEngineStartTimestampMetric(stats);
+    }
+}
\ No newline at end of file
-- 
cgit 1.2.3-korg