Add passthrough attributes

Added passthrough attributes in placement request.
Current attributes HAS request section changed to
filtering_attributes and new passthrough attributes
added in HAS request - this one is read from vnf policy
passthroughAttributes section.

Change-Id: Ic1e9dcafb0aa0ce5c1b4ddcf35d034d457ac7d48
Signed-off-by: vrvarma <vv8305@att.com>
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@orange.com>
Issue-ID: OPTFRA-611

10 files changed, 212 insertions, 143 deletions

diff --git a/apps/placement/optimizers/conductor/translation.py b/apps/placement/optimizers/conductor/translation.py
index d361755..46bee1d 100644
--- a/apps/placement/optimizers/conductor/translation.py
+++ b/apps/placement/optimizers/conductor/translation.py
@@ -17,10 +17,10 @@
 #
 import copy
 import json
-import yaml
 import re
 
-from osdf.utils.data_conversion import text_to_symbol
+import yaml
+
 from osdf.utils.programming_utils import dot_notation
 
 policy_config_mapping = yaml.safe_load(open('config/has_config.yaml')).get('policy_config_mapping')
@@ -157,7 +157,7 @@ def gen_attribute_policy(vnf_list, attribute_policy):
     cur_policies, related_policies = gen_policy_instance(vnf_list, attribute_policy, rtype=None)
     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
         properties = p_main['content']['cloudAttributeProperty']
-        attribute_mapping = policy_config_mapping['attributes']  # wanted attributes and mapping
+        attribute_mapping = policy_config_mapping['filtering_attributes']  # wanted attributes and mapping
         p_new[p_main['content']['identity']]['properties'] = {
             'evaluate': dict((k, properties.get(attribute_mapping.get(k))) for k in attribute_mapping.keys())
         }
@@ -231,42 +231,47 @@ def get_demand_properties(demand, policies):
 
         prop.update({'unique': policy_property['unique']} if 'unique' in policy_property and
                                                              policy_property['unique'] else {})
-        prop['attributes'] = dict()
-        prop['attributes'].update({'global-customer-id': policy_property['customerId']}
+        prop['filtering_attributes'] = dict()
+        prop['filtering_attributes'].update({'global-customer-id': policy_property['customerId']}
                                   if policy_property['customerId'] else {})
-        prop['attributes'].update({'model-invariant-id': demand['resourceModelInfo']['modelInvariantId']}
+        prop['filtering_attributes'].update({'model-invariant-id': demand['resourceModelInfo']['modelInvariantId']}
                                   if demand['resourceModelInfo']['modelInvariantId'] else {})
-        prop['attributes'].update({'model-version-id': demand['resourceModelInfo']['modelVersionId']}
+        prop['filtering_attributes'].update({'model-version-id': demand['resourceModelInfo']['modelVersionId']}
                                   if demand['resourceModelInfo']['modelVersionId'] else {})
-        prop['attributes'].update({'equipment-role': policy_property['equipmentRole']}
+        prop['filtering_attributes'].update({'equipment-role': policy_property['equipmentRole']}
                                   if policy_property['equipmentRole'] else {})
 
         if policy_property.get('attributes'):
             for attr_key, attr_val in policy_property['attributes'].items():
-                update_converted_attribute(attr_key, attr_val, prop)
+                update_converted_attribute(attr_key, attr_val, prop, 'filtering_attributes')
+        if policy_property.get('passthroughAttributes'):
+            prop['passthrough_attributes'] = dict()
+            for attr_key, attr_val in policy_property['passthroughAttributes'].items():
+                update_converted_attribute(attr_key, attr_val, prop, 'passthrough_attributes')
 
         prop.update(get_candidates_demands(demand))
         demand_properties.append(prop)
     return demand_properties
 
 
-def update_converted_attribute(attr_key, attr_val, properties):
+def update_converted_attribute(attr_key, attr_val, properties, attribute_type):
     """
     Updates dictonary of attributes with one specified in the arguments.
     Automatically translates key namr from camelCase to hyphens
+    :param attribute_type: attribute section name
     :param attr_key: key of the attribute
     :param attr_val: value of the attribute
     :param properties: dictionary with attributes to update
     :return:
     """
     if attr_val:
-        remapping = policy_config_mapping['attributes']
+        remapping = policy_config_mapping[attribute_type]
         if remapping.get(attr_key):
             key_value = remapping.get(attr_key)
         else:
             key_value = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', attr_key)
             key_value = re.sub('([a-z0-9])([A-Z])', r'\1-\2', key_value).lower()
-        properties['attributes'].update({key_value: attr_val})
+        properties[attribute_type].update({key_value: attr_val})
 
 
 def gen_demands(req_json, vnf_policies):
diff --git a/config/has_config.yaml b/config/has_config.yaml
index 38a4781..2371508 100644
--- a/config/has_config.yaml
+++ b/config/has_config.yaml
@@ -1,5 +1,5 @@
 policy_config_mapping:
-    attributes:
+    filtering_attributes:
         hypervisor: hypervisor
         cloudVersion: cloud_version
         cloudType: cloud_type
@@ -19,6 +19,7 @@ policy_config_mapping:
         cloudRegionId: cloud-region-id
         orchestrationStatus: orchestration-status
         provStatus: prov-status
+    passthrough_attributes: {}
     candidates:
         # for (k1, v1), if k1 is in demand, set prop[k2] = _get_candidates(demand[k1])
         excludedCandidates: excluded_candidates
diff --git a/osdf/models/policy/placement/tosca/vnfPolicy-v20181031.yml b/osdf/models/policy/placement/tosca/vnfPolicy-v20181031.yml
index 46d8c32..8eaf178 100644
--- a/osdf/models/policy/placement/tosca/vnfPolicy-v20181031.yml
+++ b/osdf/models/policy/placement/tosca/vnfPolicy-v20181031.yml
@@ -69,3 +69,17 @@ data_types:
             unique:
                 type: string
                 required: false
+            attributes:
+                type: list
+                required: false
+                entry_schema:
+                    type:policy.data.vnfProperties_filteringAttributes
+            passthroughAttributes:
+                type: list
+                required: false
+                entry_schema:
+                    type:policy.data.vnfProperties_passthroughAttributes
+    policy.data.vnfProperties_filteringAttributes:
+        derived_from: tosca.nodes.Root
+    policy.data.vnfProperties_passthroughAttributes:
+        derived_from: tosca.nodes.Root
diff --git a/osdf/models/policy/placement/tosca_upload/onap.policies.optimization.VnfPolicy.yaml b/osdf/models/policy/placement/tosca_upload/onap.policies.optimization.VnfPolicy.yaml
index e1ec36d..e242a92 100644
--- a/osdf/models/policy/placement/tosca_upload/onap.policies.optimization.VnfPolicy.yaml
+++ b/osdf/models/policy/placement/tosca_upload/onap.policies.optimization.VnfPolicy.yaml
@@ -73,4 +73,18 @@ data_types:
             unique:
                 type: string
                 required: false
+            attributes:
+                type: list
+                required: false
+                entry_schema:
+                    type:policy.data.vnfProperties_filteringAttributes
+            passthroughAttributes:
+                type: list
+                required: false
+                entry_schema:
+                    type:policy.data.vnfProperties_passthroughAttributes
+    policy.data.vnfProperties_filteringAttributes:
+        derived_from: tosca.nodes.Root
+    policy.data.vnfProperties_passthroughAttributes:
+        derived_from: tosca.nodes.Root
 
diff --git a/test/functest/simulators/build_sim_image.sh b/test/functest/simulators/build_sim_image.sh
index 8efb273..6d6cb13 100755
--- a/test/functest/simulators/build_sim_image.sh
+++ b/test/functest/simulators/build_sim_image.sh
@@ -32,6 +32,7 @@ cp $SIMULATORS_DIR/Dockerfile $DOCKER_DIR/.
 cp -r $OSDF_DIR/osdf $DOCKER_DIR/sim
 mkdir -p $DOCKER_DIR/sim/config/
 cp $SIMULATORS_DIR/simulated-config/*.yaml $DOCKER_DIR/sim/config/
+cp $SIMULATORS_DIR/simulated-config/*.yml $DOCKER_DIR/sim/config/
 cp $SIMULATORS_DIR/simulated-config/*.config $DOCKER_DIR/sim/config/
 cp -r $SIMULATORS_DIR/configdb $DOCKER_DIR/sim
 cp -r $SIMULATORS_DIR/has-api $DOCKER_DIR/sim
diff --git a/test/placement-tests/request_placement_vfmod.json b/test/placement-tests/request_placement_vfmod.json
index e9f1966..4b2b852 100644
--- a/test/placement-tests/request_placement_vfmod.json
+++ b/test/placement-tests/request_placement_vfmod.json
@@ -1,92 +1,113 @@
 {
-    "name": "de4f04e3-0a65-470b-9d07-8ea6c2fb3e10",
-    "template": {
-        "constraints": {
-            "affinity_vFW_TD": {
-                "demands": ["vFW-SINK", "vPGN"],
-                "properties": {
-                    "category": "region",
-                    "qualifier": "same"
-                },
-                "type": "zone"
-            }
-        },
-        "parameters": {
-            "service_name": "vFW_TD",
-            "chosen_region": "RegionOne",
-            "service_id": "3e8d118c-10ca-4b4b-b3db-089b5e9e6a1c",
-            "customer_long": 2.2,
-            "REQUIRED_MEM": "",
-            "customer_lat": 1.1,
-            "REQUIRED_DISK": ""
+  "name": "de4f04e3-0a65-470b-9d07-8ea6c2fb3e10",
+  "files": {},
+  "timeout": 1200,
+  "num_solution": "100",
+  "template": {
+    "homing_template_version": "2017-10-10",
+    "parameters": {
+      "REQUIRED_MEM": "",
+      "REQUIRED_DISK": "",
+      "customer_lat": 1.1,
+      "customer_long": 2.2,
+      "service_name": "vFW_TD",
+      "service_id": "3e8d118c-10ca-4b4b-b3db-089b5e9e6a1c",
+      "chosen_region": "RegionOne"
+    },
+    "locations": {
+      "customer_loc": {
+        "latitude": {
+          "get_param": "customer_lat"
         },
-        "locations": {
-            "customer_loc": {
-                "longitude": {
-                    "get_param": "customer_long"
-                },
-                "latitude": {
-                    "get_param": "customer_lat"
-                }
+        "longitude": {
+          "get_param": "customer_long"
+        }
+      }
+    },
+    "demands": {
+      "vFW-SINK": [
+        {
+          "inventory_provider": "aai",
+          "inventory_type": "vfmodule",
+          "service_type": "vFW-SINK-XX",
+          "service_resource_id": "vFW-SINK-XX",
+          "filtering_attributes": {
+            "global-customer-id": {
+              "get_param": "chosen_customer_id"
+            },
+            "model-invariant-id": "e7227847-dea6-4374-abca-4561b070fe7d",
+            "model-version-id": "763731df-84fd-494b-b824-01fc59a5ff2d",
+            "orchestration-status": [
+              "active"
+            ],
+            "prov-status": "ACTIVE",
+            "cloud-region-id": {
+              "get_param": "chosen_region"
+            },
+            "service_instance_id": {
+              "get_param": "service_id"
             }
-        },
-        "demands": {
-            "vFW-SINK": [{
-                "attributes": {
-                    "global-customer-id": {
-                        "get_param": "chosen_customer_id"
-                    },
-                    "cloud-region-id": {
-                        "get_param": "chosen_region"
-                    },
-                    "model-version-id": "763731df-84fd-494b-b824-01fc59a5ff2d",
-                    "orchestration-status": ["active"],
-                    "model-invariant-id": "e7227847-dea6-4374-abca-4561b070fe7d",
-                    "service_instance_id": {
-                        "get_param": "service_id"
-                    },
-                    "prov-status": "ACTIVE"
-                },
-                "inventory_provider": "aai",
-                "service_resource_id": "vFW-SINK-XX",
-                "inventory_type": "vfmodule",
-                "service_type": "vFW-SINK-XX",
-                "excluded_candidates": [{
-                    "inventory_type": "vfmodule",
-                    "candidate_id": ["e765d576-8755-4145-8536-0bb6d9b1dc9a"]
-                }]
-            }],
-            "vPGN": [{
-                "attributes": {
-                    "global-customer-id": {
-                        "get_param": "chosen_customer_id"
-                    },
-                    "cloud-region-id": {
-                        "get_param": "chosen_region"
-                    },
-                    "model-version-id": "e02a7e5c-9d27-4360-ab7c-73bb83b07e3b",
-                    "orchestration-status": ["active"],
-                    "model-invariant-id": "762472ef-5284-4daa-ab32-3e7bee2ec355",
-                    "service_instance_id": {
-                        "get_param": "service_id"
-                    },
-                    "prov-status": "ACTIVE"
-                },
-                "inventory_provider": "aai",
-                "service_resource_id": "vPGN-XX",
-                "unique": "False",
-                "inventory_type": "vfmodule",
-                "service_type": "vPGN-XX"
-            }]
-        },
-        "optimization": {
-            "minimize": {
-                "sum": []
+          },
+          "passthrough_attributes": {
+            "td-role": "destination"
+          },
+          "excluded_candidates": [
+            {
+              "inventory_type": "vfmodule",
+              "candidate_id": [
+                "e765d576-8755-4145-8536-0bb6d9b1dc9a"
+              ]
             }
-        },
-        "homing_template_version": "2017-10-10"
+          ]
+        }
+      ],
+      "vPGN": [
+        {
+          "inventory_provider": "aai",
+          "inventory_type": "vfmodule",
+          "service_type": "vPGN-XX",
+          "service_resource_id": "vPGN-XX",
+          "unique": "False",
+          "filtering_attributes": {
+            "global-customer-id": {
+              "get_param": "chosen_customer_id"
+            },
+            "model-invariant-id": "762472ef-5284-4daa-ab32-3e7bee2ec355",
+            "model-version-id": "e02a7e5c-9d27-4360-ab7c-73bb83b07e3b",
+            "orchestration-status": [
+              "active"
+            ],
+            "prov-status": "ACTIVE",
+            "cloud-region-id": {
+              "get_param": "chosen_region"
+            },
+            "service_instance_id": {
+              "get_param": "service_id"
+            }
+          },
+          "passthrough_attributes": {
+            "td-role": "anchor"
+          }
+        }
+      ]
+    },
+    "constraints": {
+      "affinity_vFW_TD": {
+        "type": "zone",
+        "demands": [
+          "vFW-SINK",
+          "vPGN"
+        ],
+        "properties": {
+          "category": "region",
+          "qualifier": "same"
+        }
+      }
     },
-    "num_solution": "100",
-    "files": {},
-    "timeout": 1200
-}
+    "optimization": {
+      "minimize": {
+        "sum": []
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/test/policy-local-files/vnfPolicy_vFW_TD.json b/test/policy-local-files/vnfPolicy_vFW_TD.json
index 9a9cbe0..a471a77 100644
--- a/test/policy-local-files/vnfPolicy_vFW_TD.json
+++ b/test/policy-local-files/vnfPolicy_vFW_TD.json
@@ -35,7 +35,10 @@
                 "service_instance_id": {
                     "get_param": "service_id"
                 }
+            },
+            "passthroughAttributes": {
+                "td-role": "destination"
             }
         }]
     }
-}
\ No newline at end of file
+}
diff --git a/test/policy-local-files/vnfPolicy_vPGN_TD.json b/test/policy-local-files/vnfPolicy_vPGN_TD.json
index 3724f8a..2e79f2f 100644
--- a/test/policy-local-files/vnfPolicy_vPGN_TD.json
+++ b/test/policy-local-files/vnfPolicy_vPGN_TD.json
@@ -1,42 +1,51 @@
 {
-    "service": "vnfPolicy",
-    "policyName": "OSDF_DUBLIN.vnfPolicy_vPGN_TD",
-    "description": "vnfPolicy",
-    "templateVersion": "OpenSource.version.1",
-    "version": "oofDublin",
-    "priority": "6",
-    "riskType": "test",
-    "riskLevel": "3",
-    "guard": "False",
-    "content": {
-        "identity": "vnf_vPGN_TD",
-        "policyScope": [
-            "td",
-            "us",
-            "vPGN"
-        ],
-        "policyType": "vnfPolicy",
-        "resources": ["vPGN"],
-        "applicableResources": "any",
-        "vnfProperties": [{
-            "inventoryProvider": "aai",
-            "serviceType": "",
-            "inventoryType": "vfmodule",
-            "customerId": {
-                "get_param": "chosen_customer_id"
-            },
-            "equipmentRole": "",
-            "unique": "False",
-            "attributes": {
-                "orchestrationStatus": ["active"],
-                "provStatus": "ACTIVE",
-                "cloudRegionId": {
-                    "get_param": "chosen_region"
-                },
-                "service_instance_id": {
-                    "get_param": "service_id"
-                }
-            }
-        }]
-    }
+  "service": "vnfPolicy",
+  "policyName": "OSDF_DUBLIN.vnfPolicy_vPGN_TD",
+  "description": "vnfPolicy",
+  "templateVersion": "OpenSource.version.1",
+  "version": "oofDublin",
+  "priority": "6",
+  "riskType": "test",
+  "riskLevel": "3",
+  "guard": "False",
+  "content": {
+    "identity": "vnf_vPGN_TD",
+    "policyScope": [
+      "td",
+      "us",
+      "vPGN"
+    ],
+    "policyType": "vnfPolicy",
+    "resources": [
+      "vPGN"
+    ],
+    "applicableResources": "any",
+    "vnfProperties": [
+      {
+        "inventoryProvider": "aai",
+        "serviceType": "",
+        "inventoryType": "vfmodule",
+        "customerId": {
+          "get_param": "chosen_customer_id"
+        },
+        "equipmentRole": "",
+        "unique": "False",
+        "attributes": {
+          "orchestrationStatus": [
+            "active"
+          ],
+          "provStatus": "ACTIVE",
+          "cloudRegionId": {
+            "get_param": "chosen_region"
+          },
+          "service_instance_id": {
+            "get_param": "service_id"
+          }
+        },
+        "passthroughAttributes": {
+          "td-role": "anchor"
+        }
+      }
+    ]
+  }
 }
\ No newline at end of file
diff --git a/test/test-requirements.txt b/test/test-requirements.txt
index 043d87a..c8d5613 100644
--- a/test/test-requirements.txt
+++ b/test/test-requirements.txt
@@ -3,3 +3,4 @@ moto
 pytest
 pytest-tap
 requests-mock
+pylint
diff --git a/test/test_ConductorApiBuilder.py b/test/test_ConductorApiBuilder.py
index e69e954..07cb3bb 100644
--- a/test/test_ConductorApiBuilder.py
+++ b/test/test_ConductorApiBuilder.py
@@ -47,7 +47,7 @@ class TestConductorApiBuilder(unittest.TestCase):
         main_dir = self.main_dir
         request_json = self.request_json
         policies = self.policies
-        local_config = yaml.load(open(self.local_config_file))
+        local_config = yaml.safe_load(open(self.local_config_file))
         templ_string = conductor_api_builder(request_json, policies, local_config, self.conductor_api_template)
         templ_json = json.loads(templ_string)
         self.assertEqual(templ_json["name"], "yyy-yyy-yyyy")
@@ -55,7 +55,7 @@ class TestConductorApiBuilder(unittest.TestCase):
     def test_conductor_api_call_builder_vfmod(self):
         request_json = self.request_vfmod_json
         policies = self.policies
-        local_config = yaml.load(open(self.local_config_file))
+        local_config = yaml.safe_load(open(self.local_config_file))
         templ_string = conductor_api_builder(request_json, policies, local_config, self.conductor_api_template)
         templ_json = json.loads(templ_string)
         self.assertEqual(templ_json, self.request_placement_vfmod_json)