Reimplement Lock API using Lock objects

Modified PolicyResourceLockManager to just return a feature, deferring
the lock() call/method to the feature, itself.  The manager was also
modified so that, if it can't find an enabled provider, it will return
a default provider, whose lock() methods always fail.  Once a feature
has been identified, the manager will cache it for use thereafter.

Modified the feature API to return lock objects and simplified the
interface to remove the beforeXxx and afterXxx methods.
Moved the unlock and refresh methods from the feature API into the
lock class, renaming them to free and extend, respectively.
Added a separate, feature-simple-locking project, which implements
a simple version of the locking feature, over a single JVM.
Extensively revised the distributed locking feature to fit in with
the new API.

Added support for persistence so that the various LockImpl classes
can be serialized and still function correctly when they are
deserialized back into new feature instances
Added default implementations of free & extend to LockImpl.
Modified API to take the ownerKey string, instead of the owner object.
Removed Extractor as unneeded - may add via another review, if still useful.

Updates per review comments:
- Updated licenses in feature-simple-locking
- Added beforeCreateLock & afterCreateLock to feature API
- Moved SimpleLockingFeature into policy-management so that it's
  always available
- Moved the executor service, "exsvc", into PolicyEngine
- Moved Extrator into policy-utils
- Changed Extractor logging level for exceptions
- Fixed feature sequence numbers
- Fixed mixing of seconds and milliseconds
- Renamed exsvc
- Modified to use property method with default value
- Configured scheduled executor
- Added suffix to Extractor.register()
- Eliminated Feature Api and tied lock manager into engine
- Added non-null checks to LockImpl parameters
- Added non-null checks to createLock() parameters
- Checked that lockManager is initialized

Change-Id: Iddba38157ddc5f7277656979c0e679e5489eb7b1
Issue-ID: POLICY-2113
Signed-off-by: Jim Hahn <jrh3@att.com>

6 files changed, 704 insertions, 3 deletions

diff --git a/policy-management/src/main/java/org/onap/policy/drools/features/PolicyEngineFeatureApi.java b/policy-management/src/main/java/org/onap/policy/drools/features/PolicyEngineFeatureApi.java
index fe31eb50..87001ad6 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/features/PolicyEngineFeatureApi.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/features/PolicyEngineFeatureApi.java
@@ -23,6 +23,7 @@ package org.onap.policy.drools.features;
 import java.util.Properties;
 import org.onap.policy.common.endpoints.event.comm.Topic.CommInfrastructure;
 import org.onap.policy.common.utils.services.OrderedService;
+import org.onap.policy.drools.core.lock.PolicyResourceLockManager;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
 import org.onap.policy.drools.system.PolicyEngine;
 
@@ -271,4 +272,26 @@ public interface PolicyEngineFeatureApi extends OrderedService {
     default boolean afterOpen(PolicyEngine engine) {
         return false;
     }
+
+    /**
+     * Called before the PolicyEngine creates a lock manager.
+     *
+     * @return a lock manager if this feature intercepts and takes ownership of the
+     *         operation preventing the invocation of lower priority features. Null,
+     *         otherwise
+     */
+    default PolicyResourceLockManager beforeCreateLockManager(PolicyEngine engine, Properties properties) {
+        return null;
+    }
+
+    /**
+     * Called after the PolicyEngine creates a lock manager.
+     *
+     * @return True if this feature intercepts and takes ownership of the operation
+     *         preventing the invocation of lower priority features. False, otherwise
+     */
+    default boolean afterCreateLockManager(PolicyEngine engine, Properties properties,
+                    PolicyResourceLockManager lockManager) {
+        return false;
+    }
 }
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
index 653ff72e..cb0749d9 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
@@ -22,6 +22,7 @@ package org.onap.policy.drools.system;
 
 import java.util.List;
 import java.util.Properties;
+import java.util.concurrent.ScheduledExecutorService;
 import org.onap.policy.common.capabilities.Lockable;
 import org.onap.policy.common.capabilities.Startable;
 import org.onap.policy.common.endpoints.event.comm.Topic.CommInfrastructure;
@@ -29,6 +30,8 @@ import org.onap.policy.common.endpoints.event.comm.TopicListener;
 import org.onap.policy.common.endpoints.event.comm.TopicSink;
 import org.onap.policy.common.endpoints.event.comm.TopicSource;
 import org.onap.policy.common.endpoints.http.server.HttpServletServer;
+import org.onap.policy.drools.core.lock.Lock;
+import org.onap.policy.drools.core.lock.LockCallback;
 import org.onap.policy.drools.features.PolicyEngineFeatureApi;
 import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
@@ -198,6 +201,11 @@ public interface PolicyEngine extends Startable, Lockable, TopicListener {
     List<HttpServletServer> getHttpServers();
 
     /**
+     * Gets a thread pool that can be used to execute background tasks.
+     */
+    ScheduledExecutorService getExecutorService();
+
+    /**
      * get properties configuration.
      *
      * @return properties objects
@@ -280,6 +288,31 @@ public interface PolicyEngine extends Startable, Lockable, TopicListener {
     boolean deliver(CommInfrastructure busType, String topic, String event);
 
     /**
+     * Requests a lock on a resource. Typically, the lock is not immediately granted,
+     * though a "lock" object is always returned. Once the lock has been granted (or
+     * denied), the callback will be invoked to indicate the result.
+     *
+     * <p/>
+     * Notes:
+     * <dl>
+     * <li>The callback may be invoked <i>before</i> this method returns</li>
+     * <li>The implementation need not honor waitForLock={@code true}</li>
+     * </dl>
+     *
+     * @param resourceId identifier of the resource to be locked
+     * @param ownerKey information identifying the owner requesting the lock
+     * @param holdSec amount of time, in seconds, for which the lock should be held once
+     *        it has been granted, after which it will automatically be released
+     * @param callback callback to be invoked once the lock is granted, or subsequently
+     *        lost; must not be {@code null}
+     * @param waitForLock {@code true} to wait for the lock, if it is currently locked,
+     *        {@code false} otherwise
+     * @return a new lock
+     */
+    public Lock createLock(String resourceId, String ownerKey, int holdSec, LockCallback callback,
+                    boolean waitForLock);
+
+    /**
      * Invoked when the host goes into the active state.
      */
     void activate();
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
index 766848c6..c924fd69 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngineManager.java
@@ -24,7 +24,6 @@ import static org.onap.policy.drools.system.PolicyEngineConstants.TELEMETRY_SERV
 import static org.onap.policy.drools.system.PolicyEngineConstants.TELEMETRY_SERVER_DEFAULT_NAME;
 import static org.onap.policy.drools.system.PolicyEngineConstants.TELEMETRY_SERVER_DEFAULT_PORT;
 
-import com.att.aft.dme2.internal.apache.commons.lang3.StringUtils;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.gson.Gson;
@@ -32,11 +31,16 @@ import com.google.gson.GsonBuilder;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
+import lombok.AccessLevel;
 import lombok.Getter;
+import lombok.NonNull;
+import org.apache.commons.lang.StringUtils;
 import org.onap.policy.common.endpoints.event.comm.Topic;
 import org.onap.policy.common.endpoints.event.comm.Topic.CommInfrastructure;
 import org.onap.policy.common.endpoints.event.comm.TopicEndpoint;
@@ -54,6 +58,9 @@ import org.onap.policy.drools.controller.DroolsController;
 import org.onap.policy.drools.controller.DroolsControllerConstants;
 import org.onap.policy.drools.core.PolicyContainer;
 import org.onap.policy.drools.core.jmx.PdpJmxListener;
+import org.onap.policy.drools.core.lock.Lock;
+import org.onap.policy.drools.core.lock.LockCallback;
+import org.onap.policy.drools.core.lock.PolicyResourceLockManager;
 import org.onap.policy.drools.features.PolicyControllerFeatureApi;
 import org.onap.policy.drools.features.PolicyControllerFeatureApiConstants;
 import org.onap.policy.drools.features.PolicyEngineFeatureApi;
@@ -67,6 +74,7 @@ import org.onap.policy.drools.protocol.configuration.ControllerConfiguration;
 import org.onap.policy.drools.protocol.configuration.PdpdConfiguration;
 import org.onap.policy.drools.server.restful.RestManager;
 import org.onap.policy.drools.server.restful.aaf.AafTelemetryAuthFilter;
+import org.onap.policy.drools.system.internal.SimpleLockManager;
 import org.onap.policy.drools.utils.PropertyUtil;
 import org.onap.policy.drools.utils.logging.LoggerUtil;
 import org.onap.policy.drools.utils.logging.MdcTransaction;
@@ -78,7 +86,6 @@ import org.slf4j.LoggerFactory;
  * Policy Engine Manager Implementation.
  */
 class PolicyEngineManager implements PolicyEngine {
-
     /**
      * String literals.
      */
@@ -88,6 +95,9 @@ class PolicyEngineManager implements PolicyEngine {
     private static final String ENGINE_STOPPED_MSG = "Policy Engine is stopped";
     private static final String ENGINE_LOCKED_MSG = "Policy Engine is locked";
 
+    public static final String EXECUTOR_THREAD_PROP = "executor.threads";
+    protected static final int DEFAULT_EXECUTOR_THREADS = 5;
+
     /**
      * logger.
      */
@@ -134,6 +144,17 @@ class PolicyEngineManager implements PolicyEngine {
     private List<HttpServletServer> httpServers = new ArrayList<>();
 
     /**
+     * Thread pool used to execute background tasks.
+     */
+    private ScheduledExecutorService executorService = null;
+
+    /**
+     * Lock manager used to create locks.
+     */
+    @Getter(AccessLevel.PROTECTED)
+    private PolicyResourceLockManager lockManager = null;
+
+    /**
      * gson parser to decode configuration requests.
      */
     private final Gson decoder = new GsonBuilder().disableHtmlEscaping().create();
@@ -214,6 +235,53 @@ class PolicyEngineManager implements PolicyEngine {
     }
 
     @Override
+    @JsonIgnore
+    @GsonJsonIgnore
+    public ScheduledExecutorService getExecutorService() {
+        return executorService;
+    }
+
+    private ScheduledExecutorService makeExecutorService(Properties properties) {
+        int nthreads = DEFAULT_EXECUTOR_THREADS;
+        try {
+            nthreads = Integer.valueOf(
+                            properties.getProperty(EXECUTOR_THREAD_PROP, String.valueOf(DEFAULT_EXECUTOR_THREADS)));
+
+        } catch (NumberFormatException e) {
+            logger.error("invalid number for " + EXECUTOR_THREAD_PROP + " property", e);
+        }
+
+        return makeScheduledExecutor(nthreads);
+    }
+
+    private void createLockManager(Properties properties) {
+        for (PolicyEngineFeatureApi feature : getEngineProviders()) {
+            try {
+                this.lockManager = feature.beforeCreateLockManager(this, properties);
+                if (this.lockManager != null) {
+                    return;
+                }
+            } catch (RuntimeException e) {
+                logger.error("{}: feature {} before-create-lock-manager failure because of {}", this,
+                                feature.getClass().getName(), e.getMessage(), e);
+            }
+        }
+
+        try {
+            this.lockManager = new SimpleLockManager(this, properties);
+        } catch (RuntimeException e) {
+            logger.error("{}: cannot create simple lock manager because of {}", this, e.getMessage(), e);
+            this.lockManager = new SimpleLockManager(this, new Properties());
+        }
+
+        /* policy-engine dispatch post operation hook */
+        FeatureApiUtils.apply(getEngineProviders(),
+            feature -> feature.afterCreateLockManager(this, properties, this.lockManager),
+            (feature, ex) -> logger.error("{}: feature {} after-create-lock-manager failure because of {}",
+                            this, feature.getClass().getName(), ex.getMessage(), ex));
+    }
+
+    @Override
     public synchronized void configure(Properties properties) {
 
         if (properties == null) {
@@ -257,6 +325,10 @@ class PolicyEngineManager implements PolicyEngine {
             logger.error("{}: add-http-servers failed", this, e);
         }
 
+        executorService = makeExecutorService(properties);
+
+        createLockManager(properties);
+
         /* policy-engine dispatch post configure hook */
         FeatureApiUtils.apply(getEngineProviders(),
             feature -> feature.afterConfigure(this),
@@ -499,6 +571,13 @@ class PolicyEngineManager implements PolicyEngine {
 
         AtomicReference<Boolean> success = new AtomicReference<>(true);
 
+        try {
+            success.compareAndSet(true, this.lockManager.start());
+        } catch (final RuntimeException e) {
+            logger.warn("{}: cannot start lock manager because of {}", this, e.getMessage(), e);
+            success.set(false);
+        }
+
         /* Start Policy Engine exclusively-owned (unmanaged) http servers */
 
         attempt(success, this.httpServers,
@@ -639,9 +718,16 @@ class PolicyEngineManager implements PolicyEngine {
             (item, ex) -> logger.error("{}: cannot stop http-server {} because of {}", this, item,
                             ex.getMessage(), ex));
 
+        try {
+            success.compareAndSet(true, this.lockManager.stop());
+        } catch (final RuntimeException e) {
+            logger.warn("{}: cannot stop lock manager because of {}", this, e.getMessage(), e);
+            success.set(false);
+        }
+
         // stop JMX?
 
-        /* policy-engine dispatch pre stop hook */
+        /* policy-engine dispatch post stop hook */
         FeatureApiUtils.apply(getEngineProviders(),
             feature -> feature.afterStop(this),
             (feature, ex) -> logger.error("{}: feature {} after-stop failure because of {}", this,
@@ -688,6 +774,14 @@ class PolicyEngineManager implements PolicyEngine {
         getTopicEndpointManager().shutdown();
         getServletFactory().destroy();
 
+        try {
+            this.lockManager.shutdown();
+        } catch (final RuntimeException e) {
+            logger.warn("{}: cannot shutdown lock manager because of {}", this, e.getMessage(), e);
+        }
+
+        executorService.shutdownNow();
+
         // Stop the JMX listener
 
         stopPdpJmxListener();
@@ -806,6 +900,13 @@ class PolicyEngineManager implements PolicyEngine {
 
         success = getTopicEndpointManager().lock() && success;
 
+        try {
+            success = (this.lockManager == null || this.lockManager.lock()) && success;
+        } catch (final RuntimeException e) {
+            logger.warn("{}: cannot lock() lock manager because of {}", this, e.getMessage(), e);
+            success = false;
+        }
+
         /* policy-engine dispatch post lock hook */
         FeatureApiUtils.apply(getEngineProviders(),
             feature -> feature.afterLock(this),
@@ -833,6 +934,14 @@ class PolicyEngineManager implements PolicyEngine {
         this.locked = false;
 
         boolean success = true;
+
+        try {
+            success = (this.lockManager == null || this.lockManager.unlock()) && success;
+        } catch (final RuntimeException e) {
+            logger.warn("{}: cannot unlock() lock manager because of {}", this, e.getMessage(), e);
+            success = false;
+        }
+
         final List<PolicyController> controllers = getControllerFactory().inventory();
         for (final PolicyController controller : controllers) {
             try {
@@ -1167,6 +1276,21 @@ class PolicyEngineManager implements PolicyEngine {
                             feature.getClass().getName(), ex.getMessage(), ex));
     }
 
+    @Override
+    public Lock createLock(@NonNull String resourceId, @NonNull String ownerKey, int holdSec,
+                    @NonNull LockCallback callback, boolean waitForLock) {
+
+        if (holdSec < 0) {
+            throw new IllegalArgumentException("holdSec is negative");
+        }
+
+        if (lockManager == null) {
+            throw new IllegalStateException("lock manager has not been initialized");
+        }
+
+        return lockManager.createLock(resourceId, ownerKey, holdSec, callback, waitForLock);
+    }
+
     private boolean controllerConfig(PdpdConfiguration config) {
         /* only this one supported for now */
         final List<ControllerConfiguration> configControllers = config.getControllers();
@@ -1234,4 +1358,13 @@ class PolicyEngineManager implements PolicyEngine {
     protected PolicyEngine getPolicyEngine() {
         return PolicyEngineConstants.getManager();
     }
+
+    protected ScheduledExecutorService makeScheduledExecutor(int nthreads) {
+        ScheduledThreadPoolExecutor exsvc = new ScheduledThreadPoolExecutor(nthreads);
+        exsvc.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
+        exsvc.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
+        exsvc.setRemoveOnCancelPolicy(true);
+
+        return exsvc;
+    }
 }
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManager.java b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManager.java
new file mode 100644
index 00000000..f5163e9b
--- /dev/null
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManager.java
@@ -0,0 +1,426 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.system.internal;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.Setter;
+import org.onap.policy.common.utils.properties.exception.PropertyException;
+import org.onap.policy.common.utils.time.CurrentTime;
+import org.onap.policy.drools.core.lock.AlwaysFailLock;
+import org.onap.policy.drools.core.lock.Lock;
+import org.onap.policy.drools.core.lock.LockCallback;
+import org.onap.policy.drools.core.lock.LockImpl;
+import org.onap.policy.drools.core.lock.LockState;
+import org.onap.policy.drools.core.lock.PolicyResourceLockManager;
+import org.onap.policy.drools.system.PolicyEngine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Simple implementation of the Lock Feature. Locks do not span across instances of this
+ * object (i.e., locks do not span across servers).
+ *
+ * <p/>
+ * Note: this implementation does <i>not</i> honor the waitForLocks={@code true}
+ * parameter.
+ *
+ * <p/>
+ * When a lock is deserialized, it will not initially appear in this feature's map; it
+ * will be added to the map once free() or extend() is invoked, provided there isn't
+ * already an entry.
+ */
+public class SimpleLockManager implements PolicyResourceLockManager {
+
+    private static final Logger logger = LoggerFactory.getLogger(SimpleLockManager.class);
+
+    private static final String NOT_LOCKED_MSG = "not locked";
+    private static final String LOCK_LOST_MSG = "lock lost";
+
+    /**
+     * Provider of current time. May be overridden by junit tests.
+     */
+    private static CurrentTime currentTime = new CurrentTime();
+
+    @Getter(AccessLevel.PROTECTED)
+    @Setter(AccessLevel.PROTECTED)
+    private static SimpleLockManager latestInstance = null;
+
+
+    /**
+     * Engine with which this manager is associated.
+     */
+    private final PolicyEngine engine;
+
+    /**
+     * Feature properties.
+     */
+    private final SimpleLockProperties featProps;
+
+    /**
+     * Maps a resource to the lock that owns it.
+     */
+    private final Map<String, SimpleLock> resource2lock = new ConcurrentHashMap<>();
+
+    /**
+     * Thread pool used to check for lock expiration and to notify owners when locks are
+     * lost.
+     */
+    private ScheduledExecutorService exsvc = null;
+
+    /**
+     * Used to cancel the expiration checker on shutdown.
+     */
+    private ScheduledFuture<?> checker = null;
+
+
+    /**
+     * Constructs the object.
+     *
+     * @param engine engine with which this manager is associated
+     * @param properties properties used to configure the manager
+     */
+    public SimpleLockManager(PolicyEngine engine, Properties properties) {
+        try {
+            this.engine = engine;
+            this.featProps = new SimpleLockProperties(properties);
+
+        } catch (PropertyException e) {
+            throw new SimpleLockManagerException(e);
+        }
+    }
+
+    @Override
+    public boolean isAlive() {
+        return (checker != null);
+    }
+
+    @Override
+    public boolean start() {
+        if (checker != null) {
+            return false;
+        }
+
+        exsvc = getThreadPool();
+
+        checker = exsvc.scheduleWithFixedDelay(this::checkExpired, featProps.getExpireCheckSec(),
+                        featProps.getExpireCheckSec(), TimeUnit.SECONDS);
+
+        setLatestInstance(this);
+
+        return true;
+    }
+
+    /**
+     * Stops the expiration checker. Does <i>not</i> invoke any lock call-backs.
+     */
+    @Override
+    public synchronized boolean stop() {
+        exsvc = null;
+
+        if (checker == null) {
+            return false;
+        }
+
+        ScheduledFuture<?> checker2 = checker;
+        checker = null;
+
+        checker2.cancel(true);
+
+        return true;
+    }
+
+    @Override
+    public void shutdown() {
+        stop();
+    }
+
+    @Override
+    public boolean isLocked() {
+        return false;
+    }
+
+    @Override
+    public boolean lock() {
+        return true;
+    }
+
+    @Override
+    public boolean unlock() {
+        return true;
+    }
+
+    @Override
+    public Lock createLock(String resourceId, String ownerKey, int holdSec, LockCallback callback,
+                    boolean waitForLock) {
+
+        if (latestInstance != this) {
+            AlwaysFailLock lock = new AlwaysFailLock(resourceId, ownerKey, holdSec, callback);
+            lock.notifyUnavailable();
+            return lock;
+        }
+
+        SimpleLock lock = makeLock(LockState.WAITING, resourceId, ownerKey, holdSec, callback);
+
+        SimpleLock existingLock = resource2lock.putIfAbsent(resourceId, lock);
+
+        if (existingLock == null) {
+            lock.grant();
+        } else {
+            lock.deny("resource is busy");
+        }
+
+        return lock;
+    }
+
+    /**
+     * Checks for expired locks.
+     */
+    private void checkExpired() {
+        long currentMs = currentTime.getMillis();
+        logger.info("checking for expired locks at {}", currentMs);
+
+        /*
+         * Could do this via an iterator, but using compute() guarantees that the lock
+         * doesn't get extended while it's being removed from the map.
+         */
+        for (Entry<String, SimpleLock> ent : resource2lock.entrySet()) {
+            if (!ent.getValue().expired(currentMs)) {
+                continue;
+            }
+
+            AtomicReference<SimpleLock> lockref = new AtomicReference<>(null);
+
+            resource2lock.computeIfPresent(ent.getKey(), (resourceId, lock) -> {
+                if (lock.expired(currentMs)) {
+                    lockref.set(lock);
+                    return null;
+                }
+
+                return lock;
+            });
+
+            SimpleLock lock = lockref.get();
+            if (lock != null) {
+                lock.deny("lock expired");
+            }
+        }
+    }
+
+    /**
+     * Simple Lock implementation.
+     */
+    public static class SimpleLock extends LockImpl {
+        private static final long serialVersionUID = 1L;
+
+        /**
+         * Time, in milliseconds, when the lock expires.
+         */
+        @Getter
+        private long holdUntilMs;
+
+        /**
+         * Feature containing this lock. May be {@code null} until the feature is
+         * identified. Note: this can only be null if the lock has been de-serialized.
+         */
+        private transient SimpleLockManager feature;
+
+        /**
+         * Constructs the object.
+         */
+        public SimpleLock() {
+            this.holdUntilMs = 0;
+            this.feature = null;
+        }
+
+        /**
+         * Constructs the object.
+         *
+         * @param state initial state of the lock
+         * @param resourceId identifier of the resource to be locked
+         * @param ownerKey information identifying the owner requesting the lock
+         * @param holdSec amount of time, in seconds, for which the lock should be held,
+         *        after which it will automatically be released
+         * @param callback callback to be invoked once the lock is granted, or
+         *        subsequently lost; must not be {@code null}
+         * @param feature feature containing this lock
+         */
+        public SimpleLock(LockState state, String resourceId, String ownerKey, int holdSec, LockCallback callback,
+                        SimpleLockManager feature) {
+            super(state, resourceId, ownerKey, holdSec, callback);
+            this.feature = feature;
+        }
+
+        /**
+         * Determines if the owner's lock has expired.
+         *
+         * @param currentMs current time, in milliseconds
+         * @return {@code true} if the owner's lock has expired, {@code false} otherwise
+         */
+        public boolean expired(long currentMs) {
+            return (holdUntilMs <= currentMs);
+        }
+
+        /**
+         * Grants this lock. The notification is <i>always</i> invoked via a background
+         * thread.
+         */
+        protected synchronized void grant() {
+            if (isUnavailable()) {
+                return;
+            }
+
+            setState(LockState.ACTIVE);
+            holdUntilMs = currentTime.getMillis() + TimeUnit.SECONDS.toMillis(getHoldSec());
+
+            logger.info("lock granted: {}", this);
+
+            feature.exsvc.execute(this::notifyAvailable);
+        }
+
+        /**
+         * Permanently denies this lock. The notification is invoked via a background
+         * thread, if a feature instance is attached, otherwise it uses the foreground
+         * thread.
+         *
+         * @param reason the reason the lock was denied
+         */
+        protected void deny(String reason) {
+            synchronized (this) {
+                setState(LockState.UNAVAILABLE);
+            }
+
+            logger.info("{}: {}", reason, this);
+
+            if (feature == null) {
+                notifyUnavailable();
+
+            } else {
+                feature.exsvc.execute(this::notifyUnavailable);
+            }
+        }
+
+        @Override
+        public boolean free() {
+            // do a quick check of the state
+            if (isUnavailable()) {
+                return false;
+            }
+
+            logger.info("releasing lock: {}", this);
+
+            if (!attachFeature()) {
+                setState(LockState.UNAVAILABLE);
+                return false;
+            }
+
+            AtomicBoolean result = new AtomicBoolean(false);
+
+            feature.resource2lock.computeIfPresent(getResourceId(), (resourceId, curlock) -> {
+
+                if (curlock == this) {
+                    // this lock was the owner - resource is now available
+                    result.set(true);
+                    setState(LockState.UNAVAILABLE);
+                    return null;
+
+                } else {
+                    return curlock;
+                }
+            });
+
+            return result.get();
+        }
+
+        @Override
+        public void extend(int holdSec, LockCallback callback) {
+            if (holdSec < 0) {
+                throw new IllegalArgumentException("holdSec is negative");
+            }
+
+            setHoldSec(holdSec);
+            setCallback(callback);
+
+            // do a quick check of the state
+            if (isUnavailable() || !attachFeature()) {
+                deny(LOCK_LOST_MSG);
+                return;
+            }
+
+            if (feature.resource2lock.get(getResourceId()) == this) {
+                grant();
+            } else {
+                deny(NOT_LOCKED_MSG);
+            }
+        }
+
+        /**
+         * Attaches to the feature instance, if not already attached.
+         *
+         * @return {@code true} if the lock is now attached to a feature, {@code false}
+         *         otherwise
+         */
+        private synchronized boolean attachFeature() {
+            if (feature != null) {
+                // already attached
+                return true;
+            }
+
+            feature = latestInstance;
+            if (feature == null) {
+                logger.warn("no feature yet for {}", this);
+                return false;
+            }
+
+            // put this lock into the map
+            feature.resource2lock.putIfAbsent(getResourceId(), this);
+
+            return true;
+        }
+
+        @Override
+        public String toString() {
+            return "SimpleLock [state=" + getState() + ", resourceId=" + getResourceId() + ", ownerKey=" + getOwnerKey()
+                            + ", holdSec=" + getHoldSec() + ", holdUntilMs=" + holdUntilMs + "]";
+        }
+    }
+
+    // these may be overridden by junit tests
+
+    protected ScheduledExecutorService getThreadPool() {
+        return engine.getExecutorService();
+    }
+
+    protected SimpleLock makeLock(LockState waiting, String resourceId, String ownerKey, int holdSec,
+                    LockCallback callback) {
+        return new SimpleLock(waiting, resourceId, ownerKey, holdSec, callback, this);
+    }
+}
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManagerException.java b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManagerException.java
new file mode 100644
index 00000000..ff02f39e
--- /dev/null
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockManagerException.java
@@ -0,0 +1,34 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.system.internal;
+
+public class SimpleLockManagerException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructor.
+     *
+     * @param ex exception to be wrapped
+     */
+    public SimpleLockManagerException(Exception ex) {
+        super(ex);
+    }
+}
diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockProperties.java b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockProperties.java
new file mode 100644
index 00000000..0d1ca89b
--- /dev/null
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/internal/SimpleLockProperties.java
@@ -0,0 +1,52 @@
+/*
+ * ============LICENSE_START=======================================================
+ * ONAP
+ * ================================================================================
+ * Copyright (C) 2019 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+package org.onap.policy.drools.system.internal;
+
+import java.util.Properties;
+import lombok.Getter;
+import lombok.Setter;
+import org.onap.policy.common.utils.properties.BeanConfigurator;
+import org.onap.policy.common.utils.properties.Property;
+import org.onap.policy.common.utils.properties.exception.PropertyException;
+
+
+@Getter
+@Setter
+public class SimpleLockProperties {
+    public static final String PREFIX = "simple.locking.";
+    public static final String EXPIRE_CHECK_SEC = PREFIX + "expire.check.seconds";
+
+    /**
+     * Time, in seconds, to wait between checks for expired locks.
+     */
+    @Property(name = EXPIRE_CHECK_SEC, defaultValue = "900")
+    private int expireCheckSec;
+
+    /**
+     * Constructs the object, populating fields from the properties.
+     *
+     * @param props properties from which to configure this
+     * @throws PropertyException if an error occurs
+     */
+    public SimpleLockProperties(Properties props) throws PropertyException {
+        new BeanConfigurator().configureFromProperties(this, props);
+    }
+}