shutdown to guarantee termination

This method will ensure that once the shutdown sequence is
started, it terminates either gracefully or forcefully after
the grace time period expiration.

It has been occassionally observed that the shutdown
sequence gets stuck in the shutdown sequence of a subcomponent,
such as a drools applications.   This will guarantee that
the shutdown completes regardless.

Change-Id: I8379eea5f0b80fe6a5aed6da2f2bfbfcb1f24bc0
Issue-ID: POLICY-386
Signed-off-by: Jorge Hernandez <jh1730@att.com>

1 files changed, 43 insertions, 32 deletions

diff --git a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
index b2b2df6d..d60e817a 100644
--- a/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
+++ b/policy-management/src/main/java/org/onap/policy/drools/system/PolicyEngine.java
@@ -637,7 +637,7 @@ class PolicyEngineManager implements PolicyEngine {
   }
 
   @Override
-  public PolicyController updatePolicyController(ControllerConfiguration configController) {
+  public synchronized PolicyController updatePolicyController(ControllerConfiguration configController) {
 
     if (configController == null)
       throw new IllegalArgumentException("No controller configuration has been provided");
@@ -920,6 +920,45 @@ class PolicyEngineManager implements PolicyEngine {
 
   @Override
   public synchronized void shutdown() {
+    
+    /* 
+     * shutdown activity even when underlying subcomponents
+     * (features, controllers, topics, etc ..) are stuck 
+     */
+    
+    Thread exitThread = new Thread(new Runnable() {    
+      private static final long SHUTDOWN_MAX_GRACE_TIME = 30000L;
+      
+      @Override
+      public void run() {
+        try {
+          Thread.sleep(SHUTDOWN_MAX_GRACE_TIME);
+          logger.warn("{}: abnormal termination - shutdown graceful time period expiration", 
+              PolicyEngineManager.this);
+        } catch (final InterruptedException e) {
+          /* courtesy to shutdown() to allow it to return  */
+          synchronized(PolicyEngineManager.this) {}
+          logger.info("{}: finishing a graceful shutdown ", 
+              PolicyEngineManager.this, e);
+        } finally {
+          /* 
+           * shut down the Policy Engine owned http servers as the  very last thing
+           */
+          for (final HttpServletServer httpServer : PolicyEngineManager.this.getHttpServers()) {
+            try {
+              httpServer.shutdown();
+            } catch (final Exception e) {
+              logger.error("{}: cannot shutdown http-server {} because of {}", 
+                  PolicyEngineManager.this, httpServer, e.getMessage(), e);
+            }
+          }
+          
+          logger.info("{}: exit" , PolicyEngineManager.this);
+          System.exit(0);
+        }
+      }
+    });
+    exitThread.start();
 
     /* policy-engine dispatch pre shutdown hook */
     for (final PolicyEngineFeatureAPI feature : PolicyEngineFeatureAPI.providers.getList()) {
@@ -973,37 +1012,9 @@ class PolicyEngineManager implements PolicyEngine {
             feature.getClass().getName(), e.getMessage(), e);
       }
     }
-
-
-    new Thread(new Runnable() {
-      @Override
-      public void run() {
-        try {
-          Thread.sleep(5000L);
-        } catch (final InterruptedException e) {
-          logger.warn("{}: interrupted-exception while shutting down management server: ", this);
-        }
-
-        /* shutdown all unmanaged http servers */
-        for (final HttpServletServer httpServer : PolicyEngineManager.this.getHttpServers()) {
-          try {
-            httpServer.shutdown();
-          } catch (final Exception e) {
-            logger.error("{}: cannot shutdown http-server {} because of {}", this, httpServer,
-                e.getMessage(), e);
-          }
-        }
-
-        try {
-          Thread.sleep(5000L);
-        } catch (final InterruptedException e) {
-          logger.warn("{}: interrupted-exception while shutting down management server: ", this);
-          Thread.currentThread().interrupt();
-        }
-
-        System.exit(0);
-      }
-    }).start();
+    
+    exitThread.interrupt();
+    logger.info("{}: normal termination" , this);
   }
 
   @Override