[VVP] Preload Generation Enhancements and Fixes

- All values flow to preload env templates (availability
  zones were not)
- defaults.yaml should be in preload_env (includes vnf_name)
- Ensure SDC Model Identifiers are documented in VNF API format
  (ex: vnf-type, etc.)
- Ensure CSAR is used in VNF and GR API where appropriate and available
- Flag populated preload templates with _incomplete when they are not
  fully resolved
- If a value is still set to CHANGEME in the preload env, then revert
  to the original VALUE FOR from the blank preload template
- Ensure app_tests/preload_tests/sample_heat passes all vvp validations
- Added missing depedency (bandit) to requirements.txt

Change-Id: Idf1d5e6e5237debcf3e94bed5fcf7c15e41c9e82
Issue-ID: VVP-283
Signed-off-by: Lovett, Trevor <trevor.lovett@att.com>

13 files changed, 122 insertions, 45 deletions

diff --git a/ice_validator/app_tests/preload_tests/sample_env/base.env b/ice_validator/app_tests/preload_tests/sample_env/base.env
index 0650c68..d72b1ea 100644
--- a/ice_validator/app_tests/preload_tests/sample_env/base.env
+++ b/ice_validator/app_tests/preload_tests/sample_env/base.env
@@ -37,3 +37,4 @@
 parameters:
   availability_zone_0: az0
   availability_zone_1: az1
+  vnf_name: CHANGEME
diff --git a/ice_validator/app_tests/preload_tests/sample_heat/base.env b/ice_validator/app_tests/preload_tests/sample_heat/base.env
index 3784ea0..e3ef442 100644
--- a/ice_validator/app_tests/preload_tests/sample_heat/base.env
+++ b/ice_validator/app_tests/preload_tests/sample_heat/base.env
@@ -1,15 +1,10 @@
 parameters:
-
   db_image_name: db_image
-
   db_flavor_name: db_flavor
-
   lb_image_name: lb_image
-
   lb_flavor_name: lb_flavor
-
   svc_image_name: svc_image
-
   svc_flavor_name: svc_flavor
-
-  svc_count: 3
\ No newline at end of file
+  svc_count: 3
+  mgmt_image_name: mgmt_image
+  mgmt_flavor_name: mgmt_flavor
diff --git a/ice_validator/app_tests/preload_tests/sample_heat/base.yaml b/ice_validator/app_tests/preload_tests/sample_heat/base.yaml
index 327d2ee..fdf34e0 100644
--- a/ice_validator/app_tests/preload_tests/sample_heat/base.yaml
+++ b/ice_validator/app_tests/preload_tests/sample_heat/base.yaml
@@ -270,13 +270,13 @@ resources:
     type: OS::Cinder::VolumeAttachment
     properties:
       volume_id: { get_param: db_vol0_id }
-      server: { get_resource: db_server_0 }
+      instance_uuid: { get_resource: db_server_0 }
 
   db_volume_attachment_1:
     type: OS::Cinder::VolumeAttachment
     properties:
       volume_id: { get_param: db_vol1_id }
-      server: { get_resource: db_server_1 }
+      instance_uuid: { get_resource: db_server_1 }
 
   mgmt_server_0:
     type: OS::Nova::Server
diff --git a/ice_validator/app_tests/preload_tests/sample_heat/base_volume.yaml b/ice_validator/app_tests/preload_tests/sample_heat/base_volume.yaml
index 4d47766..e326357 100644
--- a/ice_validator/app_tests/preload_tests/sample_heat/base_volume.yaml
+++ b/ice_validator/app_tests/preload_tests/sample_heat/base_volume.yaml
@@ -26,7 +26,7 @@ resources:
           params:
             VNF_NAME: {get_param: vnf_name}
       volume_type: "solidfire"
-      volume_size: { get_param: volume_size }
+      size: { get_param: volume_size }
 
   db_vol1:
     type: OS::Cinder::Volume
@@ -37,7 +37,7 @@ resources:
           params:
             VNF_NAME: {get_param: vnf_name}
       volume_type: "solidfire"
-      volume_size: { get_param: volume_size }
+      size: { get_param: volume_size }
 
 outputs:
   db_vol0_id:
diff --git a/ice_validator/app_tests/preload_tests/test_environment.py b/ice_validator/app_tests/preload_tests/test_environment.py
index b627b4b..c815bb2 100644
--- a/ice_validator/app_tests/preload_tests/test_environment.py
+++ b/ice_validator/app_tests/preload_tests/test_environment.py
@@ -37,6 +37,7 @@
 from pathlib import Path
 
 import pytest
+from mock import mock
 
 from preload.environment import CloudServiceArchive, PreloadEnvironment
 
@@ -74,6 +75,10 @@ def test_csar_get_vf_module_resource_name(csar):
     assert csar.get_vf_module_resource_name("base_vIECCF") == "stark_vccf_vf"
 
 
+def test_csar_get_vnf_type(csar):
+    assert csar.get_vnf_type("base_vIECCF") == "stark_vccf_svc/stark_vccf_vf"
+
+
 def test_csar_get_vf_module_resource_name_not_found(csar):
     assert csar.get_vf_module_resource_name("unknown") is None
 
@@ -178,3 +183,14 @@ def test_preload_environment_defaults_in_module_env(env):
         "common": "ABC",
         "my_ip": "192.168.0.1",
     }
+
+
+def test_preload_environment_uses_csar(env, monkeypatch):
+    csar = mock.MagicMock(spec=CloudServiceArchive)
+    csar.get_vnf_type = mock.Mock(return_value="stark_vccf_svc/stark_vccf_vf")
+    csar.get_vf_module_model_name = mock.Mock(return_value="model_name")
+    env = env.get_environment("env_three")
+    monkeypatch.setattr(env, "csar", csar)
+    mod = env.get_module("base")
+    assert mod["vnf-type"] == "stark_vccf_svc/stark_vccf_vf"
+    assert mod["vf-module-model-name"] == "model_name"
diff --git a/ice_validator/app_tests/preload_tests/test_grapi.py b/ice_validator/app_tests/preload_tests/test_grapi.py
index 7b56440..eea1a67 100644
--- a/ice_validator/app_tests/preload_tests/test_grapi.py
+++ b/ice_validator/app_tests/preload_tests/test_grapi.py
@@ -87,12 +87,12 @@ def preload(pytestconfig, session_dir):
 
 @pytest.fixture(scope="session")
 def base(preload):
-    return load_module(preload, "base.json")
+    return load_module(preload, "base_incomplete.json")
 
 
 @pytest.fixture(scope="session")
 def incremental(preload):
-    return load_module(preload, "incremental.json")
+    return load_module(preload, "incremental_incomplete.json")
 
 
 def test_base_fields(base):
@@ -235,9 +235,17 @@ def test_incremental_networks(incremental):
 
 
 def test_preload_env_population(preload):
-    base_path = THIS_DIR / "sample_env/preloads/grapi/base.json"
+    base_path = THIS_DIR / "sample_env/preloads/grapi/base_incomplete.json"
     data = load_json(base_path)
     azs = data["input"]["preload-vf-module-topology-information"][
         "vnf-resource-assignments"
     ]["availability-zones"]["availability-zone"]
     assert azs == ["az0", "az1"]
+
+
+def test_preload_env_population_missing_value(preload):
+    base_path = THIS_DIR / "sample_env/preloads/grapi/base_incomplete.json"
+    data = load_json(base_path)
+    vnf_name = data["input"]["preload-vf-module-topology-information"][
+        "vnf-topology-identifier-structure"]["vnf-name"]
+    assert vnf_name == "VALUE FOR: vnf_name"
diff --git a/ice_validator/app_tests/preload_tests/test_vnfapi.py b/ice_validator/app_tests/preload_tests/test_vnfapi.py
index 5732335..16a3140 100644
--- a/ice_validator/app_tests/preload_tests/test_vnfapi.py
+++ b/ice_validator/app_tests/preload_tests/test_vnfapi.py
@@ -82,12 +82,12 @@ def preload(pytestconfig, session_dir):
 
 @pytest.fixture(scope="session")
 def base(preload):
-    return load_module(preload, "base.json")
+    return load_module(preload, "base_incomplete.json")
 
 
 @pytest.fixture(scope="session")
 def incremental(preload):
-    return load_module(preload, "incremental.json")
+    return load_module(preload, "incremental_incomplete.json")
 
 
 def test_base_azs(base):
@@ -187,7 +187,7 @@ def test_incremental_networks(incremental):
 
 
 def test_preload_env_population(preload):
-    base_path = THIS_DIR / "sample_env/preloads/vnfapi/base.json"
+    base_path = THIS_DIR / "sample_env/preloads/vnfapi/base_incomplete.json"
     data = load_json(base_path)
     azs = data["input"]["vnf-topology-information"]["vnf-assignments"][
         "availability-zones"
diff --git a/ice_validator/preload/environment.py b/ice_validator/preload/environment.py
index c0f357a..5d69a99 100644
--- a/ice_validator/preload/environment.py
+++ b/ice_validator/preload/environment.py
@@ -90,6 +90,15 @@ class CloudServiceArchive:
             if props.get("type") == "org.openecomp.groups.VfModule"
         }
 
+    def get_vnf_type(self, module):
+        """
+        Concatenation of service and VF instance name
+        """
+        service_name = self.service_name
+        instance_name = self.get_vf_module_resource_name(module)
+        if service_name and instance_name:
+            return "{}/{}".format(service_name, instance_name)
+
     @property
     def vf_module_resource_names(self):
         """
@@ -125,8 +134,9 @@ class CloudServiceArchive:
         def_dir = csar_dir / "Definitions"
         check(
             def_dir.exists(),
-            f"CSAR is invalid. {csar_dir.as_posix()} does not contain a "
-            f"Definitions directory.",
+            "CSAR is invalid. {} does not contain a Definitions directory.".format(
+                csar_dir.as_posix()
+            ),
         )
         return yaml_files(def_dir)
 
@@ -165,9 +175,6 @@ class CloudServiceArchive:
 
 
 class PreloadEnvironment:
-    """
-    A
-    """
 
     def __init__(self, env_dir, parent=None):
         self.base_dir = Path(env_dir)
@@ -228,6 +235,13 @@ class PreloadEnvironment:
         for m in (parent_module, self.defaults, module):
             if m:
                 result.update(m)
+        if self.csar:
+            vnf_type = self.csar.get_vnf_type(name)
+            if vnf_type:
+                result["vnf-type"] = vnf_type
+            model_name = self.csar.get_vf_module_model_name(name)
+            if model_name:
+                result["vf-module-model-name"] = model_name
         return result
 
     @property
diff --git a/ice_validator/preload/generator.py b/ice_validator/preload/generator.py
index 38a051d..456174a 100644
--- a/ice_validator/preload/generator.py
+++ b/ice_validator/preload/generator.py
@@ -38,10 +38,23 @@
 import json
 import os
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 
 import yaml
 
 
+def represent_ordered_dict(dumper, data):
+    value = []
+
+    for item_key, item_value in data.items():
+        node_key = dumper.represent_data(item_key)
+        node_value = dumper.represent_data(item_value)
+
+        value.append((node_key, node_value))
+
+    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
+
+
 def get_json_template(template_dir, template_name):
     template_name = template_name + ".json"
     with open(os.path.join(template_dir, template_name)) as f:
@@ -109,6 +122,7 @@ class AbstractPreloadGenerator(ABC):
         self.current_module_env = {}
         self.base_output_dir = base_output_dir
         self.env_cache = {}
+        self.module_incomplete = False
 
     @classmethod
     @abstractmethod
@@ -163,9 +177,19 @@ class AbstractPreloadGenerator(ABC):
 
     def replace(self, param_name, alt_message=None, single=False):
         value = self.get_param(param_name, single)
+        value = None if value == "CHANGEME" else value
         if value:
             return value
-        return alt_message or replace(param_name)
+        else:
+            self.module_incomplete = True
+            return alt_message or replace(param_name)
+
+    def start_module(self, module, env):
+        """Initialize/reset the environment for the module"""
+        self.current_module = module
+        self.current_module_env = env
+        self.module_incomplete = False
+        self.env_cache = {}
 
     def generate_environments(self, module):
         """
@@ -179,9 +203,7 @@ class AbstractPreloadGenerator(ABC):
         print("\nGenerating Preloads for {}".format(module))
         print("-" * 50)
         print("... generating blank template")
-        self.current_module = module
-        self.current_module_env = {}
-        self.env_cache = {}
+        self.start_module(module, {})
         blank_preload_dir = self.make_preload_dir(self.base_output_dir)
         self.generate_module(module, blank_preload_dir)
         self.generate_preload_env(module, blank_preload_dir)
@@ -193,12 +215,8 @@ class AbstractPreloadGenerator(ABC):
                         env.name, output_dir
                     )
                 )
-                self.env_cache = {}
-                self.current_module = module
-                self.current_module_env = env.get_module(module.label)
+                self.start_module(module, env.get_module(module.label))
                 self.generate_module(module, output_dir)
-        self.current_module = None
-        self.current_module_env = None
 
     def make_preload_dir(self, base_dir):
         path = os.path.join(base_dir, self.output_sub_dir())
@@ -206,17 +224,23 @@ class AbstractPreloadGenerator(ABC):
             os.makedirs(path, exist_ok=True)
         return path
 
-    def generate_preload_env(self, module, blank_preload_dir):
+    @staticmethod
+    def generate_preload_env(module, blank_preload_dir):
         """
         Create a .env template suitable for completing and using for
         preload generation from env files.
         """
+        yaml.add_representer(OrderedDict, represent_ordered_dict)
         output_dir = os.path.join(blank_preload_dir, "preload_env")
-        output_file = os.path.join(output_dir, "{}.env".format(module.vnf_name))
+        env_file = os.path.join(output_dir, "{}.env".format(module.vnf_name))
+        defaults_file = os.path.join(output_dir, "defaults.yaml")
         if not os.path.exists(output_dir):
             os.makedirs(output_dir, exist_ok=True)
-        with open(output_file, "w") as f:
+        with open(env_file, "w") as f:
             yaml.dump(module.env_template, f)
+        if not os.path.exists(defaults_file):
+            with open(defaults_file, "w") as f:
+                yaml.dump({"vnf_name": "CHANGEME"}, f)
 
     def get_param(self, param_name, single):
         """
diff --git a/ice_validator/preload/model.py b/ice_validator/preload/model.py
index e37c914..dba0bb5 100644
--- a/ice_validator/preload/model.py
+++ b/ice_validator/preload/model.py
@@ -37,6 +37,7 @@
 import os
 import shutil
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 
 from preload.generator import yield_by_count
 from preload.environment import PreloadEnvironment
@@ -332,11 +333,13 @@ class VnfModule(FilterBaseOutputs):
         Returns a a template .env file that can be completed to enable
         preload generation.
         """
-        params = {}
-        params["vnf-name"] = CHANGE
+        params = OrderedDict()
+        params["vnf_name"] = CHANGE
         params["vnf-type"] = CHANGE
         params["vf-module-model-name"] = CHANGE
         params["vf_module_name"] = CHANGE
+        for az in self.availability_zones:
+            params[az] = CHANGE
         for network in self.networks:
             params[network.name_param] = CHANGE
             for param in set(network.subnet_params):
diff --git a/ice_validator/preload_grapi/grapi_generator.py b/ice_validator/preload_grapi/grapi_generator.py
index 2060cb4..d75fbbd 100644
--- a/ice_validator/preload_grapi/grapi_generator.py
+++ b/ice_validator/preload_grapi/grapi_generator.py
@@ -74,7 +74,8 @@ class GrApiPreloadGenerator(AbstractPreloadGenerator):
         template = get_json_template(DATA_DIR, "preload_template")
         self._populate(template, vnf_module)
         vnf_name = vnf_module.vnf_name
-        outfile = "{}/{}.json".format(output_dir, vnf_name)
+        incomplete = "_incomplete" if self.module_incomplete else ""
+        outfile = "{}/{}{}.json".format(output_dir, vnf_name, incomplete)
         with open(outfile, "w") as f:
             json.dump(template, f, indent=4)
 
diff --git a/ice_validator/preload_vnfapi/vnfapi_generator.py b/ice_validator/preload_vnfapi/vnfapi_generator.py
index 517c789..dce1789 100644
--- a/ice_validator/preload_vnfapi/vnfapi_generator.py
+++ b/ice_validator/preload_vnfapi/vnfapi_generator.py
@@ -76,10 +76,30 @@ class VnfApiPreloadGenerator(AbstractPreloadGenerator):
     def generate_module(self, vnf_module, output_dir):
         preload = get_json_template(DATA_DIR, "preload_template")
         self._populate(preload, vnf_module)
-        outfile = "{}/{}.json".format(output_dir, vnf_module.vnf_name)
+        incomplete = "_incomplete" if self.module_incomplete else ""
+        outfile = "{}/{}{}.json".format(output_dir, vnf_module.vnf_name, incomplete)
         with open(outfile, "w") as f:
             json.dump(preload, f, indent=4)
 
+    def _populate(self, preload, vnf_module):
+        self._add_vnf_metadata(preload)
+        self._add_availability_zones(preload, vnf_module)
+        self._add_vnf_networks(preload, vnf_module)
+        self._add_vms(preload, vnf_module)
+        self._add_parameters(preload, vnf_module)
+
+    def _add_vnf_metadata(self, preload):
+        vnf_meta = preload["input"]["vnf-topology-information"]["vnf-topology-identifier"]
+        vnf_meta["vnf-name"] = self.replace("vnf_name")
+        vnf_meta["generic-vnf-type"] = self.replace(
+            "vnf-type",
+            "VALUE FOR: Concatenation of <Service Name>/"
+            "<VF Instance Name> MUST MATCH SDC",
+        )
+        vnf_meta["vnf-type"] = self.replace(
+            "vf-module-model-name", "VALUE FOR: <vfModuleModelName> from CSAR or SDC"
+        )
+
     def add_floating_ips(self, network_template, network):
         # only one floating IP is really supported, in the preload model
         # so for now we'll just use the last one.  We might revisit this
@@ -102,12 +122,6 @@ class VnfApiPreloadGenerator(AbstractPreloadGenerator):
                 )
                 network_template["ip-count-ipv6"] += 1
 
-    def _populate(self, preload, vnf_module):
-        self._add_availability_zones(preload, vnf_module)
-        self._add_vnf_networks(preload, vnf_module)
-        self._add_vms(preload, vnf_module)
-        self._add_parameters(preload, vnf_module)
-
     def _add_availability_zones(self, preload, vnf_module):
         zones = preload["input"]["vnf-topology-information"]["vnf-assignments"][
             "availability-zones"
diff --git a/requirements.txt b/requirements.txt
index a0d292d..09c73d1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -50,3 +50,4 @@ pyinstaller
 mock
 openstack-heat
 cached-property>=1.5,<1.6
+bandit