[VVP] Preload Generation Enhancements and Fixes

- All values flow to preload env templates (availability
  zones were not)
- defaults.yaml should be in preload_env (includes vnf_name)
- Ensure SDC Model Identifiers are documented in VNF API format
  (ex: vnf-type, etc.)
- Ensure CSAR is used in VNF and GR API where appropriate and available
- Flag populated preload templates with _incomplete when they are not
  fully resolved
- If a value is still set to CHANGEME in the preload env, then revert
  to the original VALUE FOR from the blank preload template
- Ensure app_tests/preload_tests/sample_heat passes all vvp validations
- Added missing depedency (bandit) to requirements.txt

Change-Id: Idf1d5e6e5237debcf3e94bed5fcf7c15e41c9e82
Issue-ID: VVP-283
Signed-off-by: Lovett, Trevor <trevor.lovett@att.com>

3 files changed, 60 insertions, 19 deletions

diff --git a/ice_validator/preload/environment.py b/ice_validator/preload/environment.py
index c0f357a..5d69a99 100644
--- a/ice_validator/preload/environment.py
+++ b/ice_validator/preload/environment.py
@@ -90,6 +90,15 @@ class CloudServiceArchive:
             if props.get("type") == "org.openecomp.groups.VfModule"
         }
 
+    def get_vnf_type(self, module):
+        """
+        Concatenation of service and VF instance name
+        """
+        service_name = self.service_name
+        instance_name = self.get_vf_module_resource_name(module)
+        if service_name and instance_name:
+            return "{}/{}".format(service_name, instance_name)
+
     @property
     def vf_module_resource_names(self):
         """
@@ -125,8 +134,9 @@ class CloudServiceArchive:
         def_dir = csar_dir / "Definitions"
         check(
             def_dir.exists(),
-            f"CSAR is invalid. {csar_dir.as_posix()} does not contain a "
-            f"Definitions directory.",
+            "CSAR is invalid. {} does not contain a Definitions directory.".format(
+                csar_dir.as_posix()
+            ),
         )
         return yaml_files(def_dir)
 
@@ -165,9 +175,6 @@ class CloudServiceArchive:
 
 
 class PreloadEnvironment:
-    """
-    A
-    """
 
     def __init__(self, env_dir, parent=None):
         self.base_dir = Path(env_dir)
@@ -228,6 +235,13 @@ class PreloadEnvironment:
         for m in (parent_module, self.defaults, module):
             if m:
                 result.update(m)
+        if self.csar:
+            vnf_type = self.csar.get_vnf_type(name)
+            if vnf_type:
+                result["vnf-type"] = vnf_type
+            model_name = self.csar.get_vf_module_model_name(name)
+            if model_name:
+                result["vf-module-model-name"] = model_name
         return result
 
     @property
diff --git a/ice_validator/preload/generator.py b/ice_validator/preload/generator.py
index 38a051d..456174a 100644
--- a/ice_validator/preload/generator.py
+++ b/ice_validator/preload/generator.py
@@ -38,10 +38,23 @@
 import json
 import os
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 
 import yaml
 
 
+def represent_ordered_dict(dumper, data):
+    value = []
+
+    for item_key, item_value in data.items():
+        node_key = dumper.represent_data(item_key)
+        node_value = dumper.represent_data(item_value)
+
+        value.append((node_key, node_value))
+
+    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
+
+
 def get_json_template(template_dir, template_name):
     template_name = template_name + ".json"
     with open(os.path.join(template_dir, template_name)) as f:
@@ -109,6 +122,7 @@ class AbstractPreloadGenerator(ABC):
         self.current_module_env = {}
         self.base_output_dir = base_output_dir
         self.env_cache = {}
+        self.module_incomplete = False
 
     @classmethod
     @abstractmethod
@@ -163,9 +177,19 @@ class AbstractPreloadGenerator(ABC):
 
     def replace(self, param_name, alt_message=None, single=False):
         value = self.get_param(param_name, single)
+        value = None if value == "CHANGEME" else value
         if value:
             return value
-        return alt_message or replace(param_name)
+        else:
+            self.module_incomplete = True
+            return alt_message or replace(param_name)
+
+    def start_module(self, module, env):
+        """Initialize/reset the environment for the module"""
+        self.current_module = module
+        self.current_module_env = env
+        self.module_incomplete = False
+        self.env_cache = {}
 
     def generate_environments(self, module):
         """
@@ -179,9 +203,7 @@ class AbstractPreloadGenerator(ABC):
         print("\nGenerating Preloads for {}".format(module))
         print("-" * 50)
         print("... generating blank template")
-        self.current_module = module
-        self.current_module_env = {}
-        self.env_cache = {}
+        self.start_module(module, {})
         blank_preload_dir = self.make_preload_dir(self.base_output_dir)
         self.generate_module(module, blank_preload_dir)
         self.generate_preload_env(module, blank_preload_dir)
@@ -193,12 +215,8 @@ class AbstractPreloadGenerator(ABC):
                         env.name, output_dir
                     )
                 )
-                self.env_cache = {}
-                self.current_module = module
-                self.current_module_env = env.get_module(module.label)
+                self.start_module(module, env.get_module(module.label))
                 self.generate_module(module, output_dir)
-        self.current_module = None
-        self.current_module_env = None
 
     def make_preload_dir(self, base_dir):
         path = os.path.join(base_dir, self.output_sub_dir())
@@ -206,17 +224,23 @@ class AbstractPreloadGenerator(ABC):
             os.makedirs(path, exist_ok=True)
         return path
 
-    def generate_preload_env(self, module, blank_preload_dir):
+    @staticmethod
+    def generate_preload_env(module, blank_preload_dir):
         """
         Create a .env template suitable for completing and using for
         preload generation from env files.
         """
+        yaml.add_representer(OrderedDict, represent_ordered_dict)
         output_dir = os.path.join(blank_preload_dir, "preload_env")
-        output_file = os.path.join(output_dir, "{}.env".format(module.vnf_name))
+        env_file = os.path.join(output_dir, "{}.env".format(module.vnf_name))
+        defaults_file = os.path.join(output_dir, "defaults.yaml")
         if not os.path.exists(output_dir):
             os.makedirs(output_dir, exist_ok=True)
-        with open(output_file, "w") as f:
+        with open(env_file, "w") as f:
             yaml.dump(module.env_template, f)
+        if not os.path.exists(defaults_file):
+            with open(defaults_file, "w") as f:
+                yaml.dump({"vnf_name": "CHANGEME"}, f)
 
     def get_param(self, param_name, single):
         """
diff --git a/ice_validator/preload/model.py b/ice_validator/preload/model.py
index e37c914..dba0bb5 100644
--- a/ice_validator/preload/model.py
+++ b/ice_validator/preload/model.py
@@ -37,6 +37,7 @@
 import os
 import shutil
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 
 from preload.generator import yield_by_count
 from preload.environment import PreloadEnvironment
@@ -332,11 +333,13 @@ class VnfModule(FilterBaseOutputs):
         Returns a a template .env file that can be completed to enable
         preload generation.
         """
-        params = {}
-        params["vnf-name"] = CHANGE
+        params = OrderedDict()
+        params["vnf_name"] = CHANGE
         params["vnf-type"] = CHANGE
         params["vf-module-model-name"] = CHANGE
         params["vf_module_name"] = CHANGE
+        for az in self.availability_zones:
+            params[az] = CHANGE
         for network in self.networks:
             params[network.name_param] = CHANGE
             for param in set(network.subnet_params):