improved message to deployment-handler

* added errored_scopes and scope_prefixes to the message
              to deployment-handler - to prevent erroneous
              removal of policies
* unit test coverage 67%

Change-Id: I97a5d2e949273b4564f95e0a1dd71ac5eca66f41
Issue-ID: DCAEGEN2-249
Signed-off-by: Alex Shatov <alexs@att.com>

3 files changed, 60 insertions, 9 deletions

diff --git a/LICENSE.txt b/LICENSE.txt
index 69d5fc1..28665aa 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,7 +1,7 @@
 /*
 * ============LICENSE_START==========================================
 * ===================================================================
-* Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+* Copyright © 2018 AT&T Intellectual Property. All rights reserved.
 * ===================================================================
 *
 * Unless otherwise specified, all software contained herein is licensed
diff --git a/deployment-handler-API.yaml b/deployment-handler-API.yaml
index eb1aed6..c4e9213 100644
--- a/deployment-handler-API.yaml
+++ b/deployment-handler-API.yaml
@@ -475,20 +475,51 @@ definitions:
 
     properties:
       catch_up:
-        description: "flag to indicate whether the request contains all the policies in PDP or not"
+        description: flag to indicate whether the request contains all the policies in PDP or not
         type: boolean
         default: false
 
       latest_policies:
-        description: "dictionary of (policy_id -> DCAEPolicy object).  In example: replace additionalProp1,2,3 with policy_id1,2,3 values"
+        description: |
+          dictionary of (policy_id -> DCAEPolicy object).
+          In example: replace additionalProp1,2,3 with policy_id1,2,3 values
         type: object
         default: {}
         additionalProperties:
           $ref: "#/definitions/DCAEPolicy"
 
       removed_policies:
-        description: "dictionary of (policy_id -> DCAEPolicy object).  In example: replace additionalProp1,2,3 with policy_id1,2,3 values"
+        description: |
+          whether policy was removed from policy-engine.
+          dictionary of (policy_id -> true).
+          In example: replace additionalProp1,2,3 with policy_id1,2,3 values
         type: object
         default: {}
         additionalProperties:
-          $ref: "#/definitions/DCAEPolicy"
+          type: boolean
+
+      errored_policies:
+        description: |
+          whether policy-engine returned an error on the policy.
+          dictionary of (policy_id -> true).
+          In example: replace additionalProp1,2,3 with policy_id1,2,3 values
+        type: object
+        default: {}
+        additionalProperties:
+          type: boolean
+
+      errored_scopes:
+        description: >
+          on cartchup - list of policy scope_prefix values on wchich
+          the policy-engine experienced an error other than not-found data.
+        type: array
+        items:
+          type: string
+
+      scope_prefixes:
+        description: >
+          on catchup - list of all scope_prefixes used by the policy-handler
+          to retrieve the policies from policy-engine.
+        type: array
+        items:
+          type: string
diff --git a/lib/policy.js b/lib/policy.js
index 89e5b6a..87280e6 100644
--- a/lib/policy.js
+++ b/lib/policy.js
@@ -43,6 +43,8 @@ function policyUpdate(req, res, next) {
         latest_policies : JSON.stringify((req.body && req.body.latest_policies) || {}),
         removed_policies : JSON.stringify((req.body && req.body.removed_policies) || {}),
         errored_policies : JSON.stringify((req.body && req.body.errored_policies) || {}),
+        errored_scopes : JSON.stringify((req.body && req.body.errored_scopes) || []),
+        scope_prefixes : JSON.stringify((req.body && req.body.scope_prefixes) || []),
         policy_deployments : {},
         updated_policy_ids : {},
         added_policy_ids : {},
@@ -55,6 +57,8 @@ function policyUpdate(req, res, next) {
                 + " latest_policies: " + policy_update.latest_policies
                 + " removed_policies: " + policy_update.removed_policies
                 + " errored_policies: " + policy_update.errored_policies
+                + " errored_scopes: " + policy_update.errored_scopes
+                + " scope_prefixes: " + policy_update.scope_prefixes
                 );
     /**
      * reply to and free up the policy_handler
@@ -64,11 +68,23 @@ function policyUpdate(req, res, next) {
     policy_update.latest_policies = JSON.parse(policy_update.latest_policies);
     policy_update.removed_policies = JSON.parse(policy_update.removed_policies);
     policy_update.errored_policies = JSON.parse(policy_update.errored_policies);
+    policy_update.errored_scopes = JSON.parse(policy_update.errored_scopes);
+    policy_update.scope_prefixes = JSON.parse(policy_update.scope_prefixes);
 
+    const is_policy_in_scopes = function(policy_id) {
+        return policy_update.scope_prefixes.some(scope_prefix => {
+            return policy_id.startsWith(scope_prefix);
+        });
+    };
+
+    const is_policy_in_errored_scopes = function(policy_id) {
+        return policy_update.errored_scopes.some(errored_scope => {
+            return policy_id.startsWith(errored_scope);
+        });
+    };
     /**
      * filter out the policies to what is deployed in components and needs updating (new policyVersion)
      */
-
     const collect_policy_deployments = function(node_instances) {
         node_instances.forEach(node_instance => {
             if (!node_instance.runtime_properties
@@ -90,10 +106,15 @@ function policyUpdate(req, res, next) {
             const deployed_policies = node_instance.runtime_properties.policies || {};
 
             Object.keys(deployed_policies).forEach(policy_id => {
+                const deployed_policy = deployed_policies[policy_id];
                 const latest_policy = policy_update.latest_policies[policy_id];
                 if (policy_update.removed_policies[policy_id]
-                  || (policy_update.catch_up && !latest_policy
-                    && !policy_update.errored_policies[policy_id])) {
+                  || (policy_update.catch_up
+                    && deployed_policy.policy_body
+                    && !latest_policy
+                    && !policy_update.errored_policies[policy_id])
+                    && !is_policy_in_errored_scopes(policy_id)
+                    && is_policy_in_scopes(policy_id)) {
                     have_policies = true;
                     deployment.removed_policy_ids[policy_id] = true;
                     policy_update.removed_policy_ids[policy_id] = true;
@@ -101,7 +122,6 @@ function policyUpdate(req, res, next) {
                     return;
                 }
 
-                const deployed_policy = deployed_policies[policy_id];
                 if (!latest_policy || !latest_policy.policy_body
                 || isNaN(latest_policy.policy_body.policyVersion)
                 || latest_policy.policy_body.policyVersion