Close timing loop-hole when YAML updated

Noticed when YAML updates are pushed close together that some
Params objects are deleted that should not be.  Fixed that by
eliminating the "active" list concept and only deleting a Params
object that has an associated cleaner.  If it has no cleaner,
then it can't be deleted, regardless of timing with rule updates.

Added more tests for rule updates, including cases to check
for event objects being retracted, as well as Params objects
being retracted when rules are deleted.

Change-Id: I6b744b29fca228022f43e9322ea149b16d097675
Issue-ID: POLICY-1248
Signed-off-by: Jim Hahn <jrh3@att.com>

5 files changed, 515 insertions, 0 deletions

diff --git a/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopEventCleanupTest.java b/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopEventCleanupTest.java
new file mode 100644
index 000000000..4ca89e1fb
--- /dev/null
+++ b/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopEventCleanupTest.java
@@ -0,0 +1,365 @@
+/*-
+ * ============LICENSE_START=======================================================
+ * demo
+ * ================================================================================
+ * Copyright (C) 2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.template.demo;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Properties;
+import java.util.UUID;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kie.api.runtime.KieSession;
+import org.onap.policy.common.endpoints.event.comm.TopicEndpoint;
+import org.onap.policy.common.endpoints.event.comm.TopicSink;
+import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.common.endpoints.properties.PolicyEndPointProperties;
+import org.onap.policy.controlloop.ControlLoopEventStatus;
+import org.onap.policy.controlloop.VirtualControlLoopEvent;
+import org.onap.policy.controlloop.eventmanager.ControlLoopEventManager;
+import org.onap.policy.controlloop.policy.ControlLoopPolicy;
+import org.onap.policy.drools.protocol.coders.EventProtocolCoder;
+import org.onap.policy.drools.protocol.coders.JsonProtocolFilter;
+import org.onap.policy.drools.system.PolicyController;
+import org.onap.policy.drools.system.PolicyEngine;
+import org.onap.policy.drools.utils.logging.LoggerUtil;
+import org.onap.policy.template.demo.Util.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Verifies that event objects are cleaned up when rules are updated. This loads
+ * <b>two</b> copies of the rule set into a single policy to ensure that the two copies
+ * interact appropriately with each other's event objects.
+ */
+public class ControlLoopEventCleanupTest {
+    private static final Logger logger = LoggerFactory.getLogger(ControlLoopEventCleanupTest.class);
+
+    /**
+     * Number of objects per control loop, including the Params object.
+     */
+    private static int CL_OBJECTS = 7;
+
+    private static final String YAML = "src/test/resources/yaml/policy_ControlLoop_EventCleanup-test.yaml";
+
+    /**
+     * YAML to be used when the first rule set is updated.
+     */
+    private static final String YAML2 = "src/test/resources/yaml/policy_ControlLoop_EventCleanup-test2.yaml";
+
+    private static final String POLICY_VERSION = "v2.0";
+
+    private static final String POLICY_NAME = "CL_CleanupTest";
+
+    private static final String POLICY_SCOPE = "type=operational";
+
+    private static final String CONTROL_LOOP_NAME = "ControlLoop-Event-Cleanup-Test";
+
+    private static final String DROOLS_TEMPLATE = "../archetype-cl-amsterdam/src/main/resources/archetype-resources/"
+                    + "src/main/resources/__closedLoopControlName__.drl";
+
+    // values specific to the second copy of the rules
+
+    private static final String YAML_B = "src/test/resources/yaml/policy_ControlLoop_EventCleanup-test-B.yaml";
+    private static final String POLICY_NAME_B = "CL_CleanupTest_B";
+    private static final String CONTROL_LOOP_NAME_B = "ControlLoop-Event-Cleanup-Test-B";
+
+    private static final String GUARD_DISABLED = "guard.disabled";
+
+    private static String saveGuardFlag;
+
+    private static KieSession kieSession;
+    private static Util.RuleSpec[] specifications;
+
+    /**
+     * Setup the simulator.
+     */
+    @BeforeClass
+    public static void setUpSimulator() {
+        LoggerUtil.setLevel(LoggerUtil.ROOT_LOGGER, "INFO");
+
+        saveGuardFlag = PolicyEngine.manager.getEnvironmentProperty(GUARD_DISABLED);
+        PolicyEngine.manager.getEnvironment().setProperty(GUARD_DISABLED, "true");
+
+        Util.setAaiProps();
+
+        PolicyEngine.manager.configure(new Properties());
+        assertTrue(PolicyEngine.manager.start());
+        Properties noopSinkProperties = new Properties();
+        noopSinkProperties.put(PolicyEndPointProperties.PROPERTY_NOOP_SINK_TOPICS, "APPC-CL,POLICY-CL-MGT");
+        noopSinkProperties.put("noop.sink.topics.APPC-CL.events", "org.onap.policy.appc.Response");
+        noopSinkProperties.put("noop.sink.topics.APPC-CL.events.custom.gson",
+                        "org.onap.policy.appc.util.Serialization,gsonPretty");
+        noopSinkProperties.put("noop.sink.topics.POLICY-CL-MGT.events",
+                        "org.onap.policy.controlloop.VirtualControlLoopNotification");
+        noopSinkProperties.put("noop.sink.topics.POLICY-CL-MGT.events.custom.gson",
+                        "org.onap.policy.controlloop.util.Serialization,gsonPretty");
+        final List<TopicSink> noopTopics = TopicEndpoint.manager.addTopicSinks(noopSinkProperties);
+
+        EventProtocolCoder.manager.addEncoder("junit.groupId", "junit.artifactId", "POLICY-CL-MGT",
+                        "org.onap.policy.controlloop.VirtualControlLoopNotification", new JsonProtocolFilter(), null,
+                        null, 1111);
+        EventProtocolCoder.manager.addEncoder("junit.groupId", "junit.artifactId", "APPC-CL",
+                        "org.onap.policy.appc.Request", new JsonProtocolFilter(), null, null, 1111);
+
+        try {
+            Util.buildAaiSim();
+
+        } catch (Exception e) {
+            logger.error("Could not create simulator", e);
+            fail("Could not create simulator");
+        }
+
+        for (TopicSink sink : noopTopics) {
+            assertTrue(sink.start());
+        }
+
+        try {
+            specifications = new Util.RuleSpec[2];
+
+            specifications[0] = new Util.RuleSpec(DROOLS_TEMPLATE, CONTROL_LOOP_NAME, POLICY_SCOPE, POLICY_NAME,
+                            POLICY_VERSION, loadYaml(YAML));
+
+            specifications[1] = new Util.RuleSpec(DROOLS_TEMPLATE, CONTROL_LOOP_NAME_B, POLICY_SCOPE, POLICY_NAME_B,
+                            POLICY_VERSION, loadYaml(YAML_B));
+
+            kieSession = Util.buildContainer(POLICY_VERSION, specifications);
+
+        } catch (IOException e) {
+            logger.error("Could not create kieSession", e);
+            fail("Could not create kieSession");
+        }
+    }
+
+    /**
+     * Tear down.
+     */
+    @AfterClass
+    public static void tearDown() {
+        kieSession.dispose();
+
+        PolicyEngine.manager.stop();
+        HttpServletServer.factory.destroy();
+        PolicyController.factory.shutdown();
+        TopicEndpoint.manager.shutdown();
+
+        if (saveGuardFlag == null) {
+            PolicyEngine.manager.getEnvironment().remove(GUARD_DISABLED);
+
+        } else {
+            PolicyEngine.manager.getEnvironment().setProperty(GUARD_DISABLED, saveGuardFlag);
+        }
+    }
+
+    @Test
+    public void test() throws IOException {
+
+        /*
+         * Let rules create Params objects.
+         */
+        kieSession.fireAllRules();
+
+        injectEvent(CONTROL_LOOP_NAME);
+        injectEvent(CONTROL_LOOP_NAME_B);
+
+        kieSession.fireAllRules();
+        List<Object> facts = getSessionObjects();
+        
+        // should have events for both control loops
+        assertEquals(2 * CL_OBJECTS, facts.size());
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME));
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME_B));
+
+        logger.info("UPDATING VERSION TO v3.0");
+        updatePolicy(YAML2, "v3.0");
+
+        /*
+         * Let rules update Params objects. The Params for the first set of rules should
+         * now be deleted and replaced with a new one, while the Params for the second set
+         * should be unchanged.
+         */
+        kieSession.fireAllRules();
+        facts = getSessionObjects();
+
+        // should only have event for second control loop + 1 Params for first control loop
+        assertEquals(CL_OBJECTS + 1, facts.size());
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME_B));
+
+        // add event for first control loop again
+        injectEvent(CONTROL_LOOP_NAME);
+        kieSession.fireAllRules();
+
+        logger.info("UPDATING VERSION TO v4.0");
+        updatePolicy(YAML, "v4.0");
+
+        /*
+         * Let rules update Params objects. The Params for the first set of rules should
+         * now be deleted and replaced with a new one, while the Params for the second set
+         * should be unchanged.
+         */
+        kieSession.fireAllRules();
+        facts = getSessionObjects();
+
+        // should only have event for second control loop + 1 Params for first control loop
+        assertEquals(CL_OBJECTS + 1, facts.size());
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME_B));
+
+        // add event for first control loop again
+        injectEvent(CONTROL_LOOP_NAME);
+        kieSession.fireAllRules();
+
+        logger.info("UPDATING VERSION TO v4.0 (i.e., unchanged)");
+        updatePolicy(YAML, "v4.0");
+
+        /*
+         * Let rules update Params objects. As the version (and YAML) are unchanged for
+         * either rule set, both Params objects should be unchanged.
+         */
+        kieSession.fireAllRules();
+        facts = getSessionObjects();
+
+        // should have events for both control loops
+        assertEquals(2 * CL_OBJECTS, facts.size());
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME));
+        assertTrue(hasEvent(facts, CONTROL_LOOP_NAME_B));
+
+        /*
+         * Now we'll delete the first rule set. That won't actually have any immediate
+         * effect, so then we'll update the second rule set, which should trigger a
+         * clean-up of both.
+         */
+        Util.RuleSpec[] specs = new Util.RuleSpec[1];
+        specs[0] = specifications[1];
+
+        logger.info("UPDATING VERSION TO v5.0 - DELETED RULE SET");
+        Util.updateContainer("v5.0", specs);
+
+        specs[0] = new Util.RuleSpec(DROOLS_TEMPLATE, CONTROL_LOOP_NAME_B, POLICY_SCOPE, POLICY_NAME_B, POLICY_VERSION,
+                        loadYaml(YAML));
+
+        logger.info("UPDATING VERSION TO v6.0 - UPDATED SECOND RULE SET");
+        Util.updateContainer("v6.0", specs);
+
+        kieSession.fireAllRules();
+        facts = getSessionObjects();
+
+        // only 1 Params should remain, for second rule set, but events should be gone
+        assertEquals(1, facts.size());
+        assertTrue(facts.stream().anyMatch(obj -> obj.toString().startsWith("Params( ")));
+    }
+
+    /**
+     * Updates the policy, changing the YAML associated with the first rule set.
+     *
+     * @param yamlFile name of the YAML file
+     * @param policyVersion policy version
+     * @throws IOException if an error occurs
+     */
+    private static void updatePolicy(String yamlFile, String policyVersion) throws IOException {
+
+        specifications[0] = new Util.RuleSpec(DROOLS_TEMPLATE, CONTROL_LOOP_NAME, POLICY_SCOPE, POLICY_NAME,
+                        policyVersion, loadYaml(yamlFile));
+
+        /*
+         * Update the policy within the container.
+         */
+        Util.updateContainer(policyVersion, specifications);
+    }
+
+    /**
+     * Loads a YAML file and URL-encodes it.
+     *
+     * @param yamlFile name of the YAML file
+     * @return the contents of the specified file, URL-encoded
+     * @throws UnsupportedEncodingException if an error occurs
+     */
+    private static String loadYaml(String yamlFile) throws UnsupportedEncodingException {
+        Pair<ControlLoopPolicy, String> pair = Util.loadYaml(yamlFile);
+        assertNotNull(pair);
+        assertNotNull(pair.first);
+        assertNotNull(pair.first.getControlLoop());
+        assertNotNull(pair.first.getControlLoop().getControlLoopName());
+        assertTrue(pair.first.getControlLoop().getControlLoopName().length() > 0);
+
+        return URLEncoder.encode(pair.second, "UTF-8");
+    }
+
+    /**
+     * Gets the session objects.
+     *
+     * @return the session objects
+     */
+    private static List<Object> getSessionObjects() {
+        // sort the objects so we know the order
+        LinkedList<Object> lst = new LinkedList<>(kieSession.getObjects());
+        lst.sort((left, right) -> left.toString().compareTo(right.toString()));
+
+        lst.forEach(obj -> logger.info("obj={}", obj));
+
+        return lst;
+    }
+
+    /**
+     * Injects an ONSET event into the rule engine.
+     *
+     * @param controlLoopName the control loop name
+     */
+    private void injectEvent(String controlLoopName) {
+        VirtualControlLoopEvent event = new VirtualControlLoopEvent();
+
+        event.setClosedLoopControlName(controlLoopName);
+
+        UUID reqid = UUID.randomUUID();
+        event.setRequestId(reqid);
+
+        event.setTarget("generic-vnf.vnf-id");
+        event.setClosedLoopAlarmStart(Instant.now());
+        event.setAai(new HashMap<>());
+        event.getAai().put("generic-vnf.vnf-id", "vnf-" + reqid.toString());
+        event.getAai().put(ControlLoopEventManager.GENERIC_VNF_IS_CLOSED_LOOP_DISABLED, "false");
+        event.setClosedLoopEventStatus(ControlLoopEventStatus.ONSET);
+
+        kieSession.insert(event);
+    }
+
+    /**
+     * Determines if the facts contain an event for the given control loop.
+     * 
+     * @param facts session facts to be checked
+     * @param controlLoopName name of the control loop of interest
+     * @return {@code true} if the facts contain an event for the given control loop,
+     *         {@code false} otherwise
+     */
+    private boolean hasEvent(List<Object> facts, String controlLoopName) {
+        return (facts.stream().anyMatch(obj -> obj instanceof VirtualControlLoopEvent
+                        && controlLoopName.equals(((VirtualControlLoopEvent) obj).getClosedLoopControlName())));
+    }
+}
diff --git a/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopParamsCleanupTest.java b/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopParamsCleanupTest.java
index 52155376d..f3c6e058d 100644
--- a/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopParamsCleanupTest.java
+++ b/controlloop/templates/template.demo/src/test/java/org/onap/policy/template/demo/ControlLoopParamsCleanupTest.java
@@ -179,6 +179,29 @@ public class ControlLoopParamsCleanupTest {
         iter = facts.iterator();
         assertTrue(iter.next() == fact3);
         assertTrue(iter.next() == fact1b);
+        
+        /*
+         * Now we'll delete the first rule set.  That won't actually have any immediate
+         * effect, so then we'll update the second rule set, which should trigger a
+         * clean-up of both.
+         */
+        Util.RuleSpec[] specs = new Util.RuleSpec[1];
+        specs[0] = specifications[1];
+
+        logger.info("UPDATING VERSION TO v5.0 - DELETED RULE SET");
+        Util.updateContainer("v5.0", specs);
+
+        specs[0] = new Util.RuleSpec(DROOLS_TEMPLATE, CONTROL_LOOP_NAME_B, POLICY_SCOPE, POLICY_NAME_B,
+                        POLICY_VERSION, loadYaml(YAML));
+        
+        logger.info("UPDATING VERSION TO v6.0 - UPDATED SECOND RULE SET");
+        Util.updateContainer("v6.0", specs);
+        
+        kieSession.fireAllRules();
+        facts = getSessionObjects();
+        assertEquals(specs.length, facts.size());
+        iter = facts.iterator();
+        assertTrue(iter.next().toString().contains(CONTROL_LOOP_NAME_B));
     }
 
     /**
diff --git a/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test-B.yaml b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test-B.yaml
new file mode 100644
index 000000000..498ef766e
--- /dev/null
+++ b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test-B.yaml
@@ -0,0 +1,41 @@
+# Copyright 2018 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+controlLoop:
+  version: 2.0.0
+  controlLoopName: ControlLoop-Event-Cleanup-Test-B
+  services:
+    - serviceInvariantUUID: 5cfe6f4a-41bc-4247-8674-ebd4b98e35cc
+      serviceUUID: 0f40bba5-986e-4b3c-803f-ddd1b7b25f24
+      serviceName: 57e66ea7-0ed6-45c7-970f
+  trigger_policy: unique-policy-id-1-modifyConfig
+  timeout: 60
+  abatement: true
+ 
+policies:
+  - id: unique-policy-id-1-modifyConfig
+    name: modify packet gen config
+    description:
+    actor: APPC
+    recipe: ModifyConfig
+    target:
+      resourceID: Eace933104d443b496b8.nodes.heat.vpg
+      type: VNF
+    retry: 0
+    timeout: 30
+    success: final_success
+    failure: final_failure
+    failure_timeout: final_failure_timeout
+    failure_retries: final_failure_retries
+    failure_exception: final_failure_exception
+    failure_guard: final_failure_guard
\ No newline at end of file
diff --git a/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test.yaml b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test.yaml
new file mode 100644
index 000000000..a19b0ef6b
--- /dev/null
+++ b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test.yaml
@@ -0,0 +1,41 @@
+# Copyright 2018 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+controlLoop:
+  version: 2.0.0
+  controlLoopName: ControlLoop-Event-Cleanup-Test
+  services:
+    - serviceInvariantUUID: 5cfe6f4a-41bc-4247-8674-ebd4b98e35cc
+      serviceUUID: 0f40bba5-986e-4b3c-803f-ddd1b7b25f24
+      serviceName: 57e66ea7-0ed6-45c7-970f
+  trigger_policy: unique-policy-id-1-modifyConfig
+  timeout: 60
+  abatement: true
+ 
+policies:
+  - id: unique-policy-id-1-modifyConfig
+    name: modify packet gen config
+    description:
+    actor: APPC
+    recipe: ModifyConfig
+    target:
+      resourceID: Eace933104d443b496b8.nodes.heat.vpg
+      type: VNF
+    retry: 0
+    timeout: 30
+    success: final_success
+    failure: final_failure
+    failure_timeout: final_failure_timeout
+    failure_retries: final_failure_retries
+    failure_exception: final_failure_exception
+    failure_guard: final_failure_guard
\ No newline at end of file
diff --git a/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test2.yaml b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test2.yaml
new file mode 100644
index 000000000..57062a47a
--- /dev/null
+++ b/controlloop/templates/template.demo/src/test/resources/yaml/policy_ControlLoop_EventCleanup-test2.yaml
@@ -0,0 +1,45 @@
+# Copyright 2018 AT&T Intellectual Property. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# This YAML must be slightly different from test.yaml.
+#
+controlLoop:
+  version: 3.0.0
+  controlLoopName: ControlLoop-Event-Cleanup-Test
+  services:
+    - serviceInvariantUUID: 5cfe6f4a-41bc-4247-8674-ebd4b98e35cc
+      serviceUUID: 0f40bba5-986e-4b3c-803f-ddd1b7b25f24
+      serviceName: 57e66ea7-0ed6-45c7-970f
+  trigger_policy: unique-policy-id-1-modifyConfig
+  timeout: 60
+  abatement: true
+ 
+policies:
+  - id: unique-policy-id-1-modifyConfig
+    name: modify packet gen config
+    description:
+    actor: APPC
+    recipe: ModifyConfig
+    target:
+      resourceID: Eace933104d443b496b8.nodes.heat.vpg
+      type: VNF
+    retry: 0
+    timeout: 30
+    success: final_success
+    failure: final_failure
+    failure_timeout: final_failure_timeout
+    failure_retries: final_failure_retries
+    failure_exception: final_failure_exception
+    failure_guard: final_failure_guard
\ No newline at end of file