[VVP] Generated completed preload from env files

User can supply an optional directory containing .env files and/or
CSAR VSP which can be used to generate populated preloads in the
requested format.

The nested directories can be used to create sub-environments that
inherit their settings from the parent directories.

Optionally, values can be specified in a defaults.yaml and they will
be used if that value is not defined in the .env file.  This is useful
if the parameter name and value will be the same in all modules.

Issue-ID: VVP-278

Change-Id: Icd9846c63463537793db908be8ce5dba13c4bda3
Signed-off-by: Lovett, Trevor <trevor.lovett@att.com>

1 files changed, 437 insertions, 0 deletions

diff --git a/ice_validator/preload/model.py b/ice_validator/preload/model.py
new file mode 100644
index 0000000..e37c914
--- /dev/null
+++ b/ice_validator/preload/model.py
@@ -0,0 +1,437 @@
+# -*- coding: utf8 -*-
+# ============LICENSE_START====================================================
+# org.onap.vvp/validation-scripts
+# ===================================================================
+# Copyright © 2019 AT&T Intellectual Property. All rights reserved.
+# ===================================================================
+#
+# Unless otherwise specified, all software contained herein is licensed
+# under the Apache License, Version 2.0 (the "License");
+# you may not use this software except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+#
+# Unless otherwise specified, all documentation contained herein is licensed
+# under the Creative Commons License, Attribution 4.0 Intl. (the "License");
+# you may not use this documentation except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#             https://creativecommons.org/licenses/by/4.0/
+#
+# Unless required by applicable law or agreed to in writing, documentation
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ============LICENSE_END============================================
+import os
+import shutil
+from abc import ABC, abstractmethod
+
+from preload.generator import yield_by_count
+from preload.environment import PreloadEnvironment
+from tests.helpers import (
+    get_param,
+    get_environment_pair,
+    prop_iterator,
+    get_output_dir,
+    is_base_module,
+    remove,
+)
+from tests.parametrizers import parametrize_heat_templates
+from tests.structures import NeutronPortProcessor, Heat
+from tests.test_environment_file_parameters import get_preload_excluded_parameters
+from tests.utils import nested_dict
+from tests.utils.vm_types import get_vm_type_for_nova_server
+from config import Config, get_generator_plugins
+
+CHANGE = "CHANGEME"
+
+
+# This is only used to fake out parametrizers
+class DummyMetafunc:
+    def __init__(self, config):
+        self.inputs = {}
+        self.config = config
+
+    def parametrize(self, name, file_list):
+        self.inputs[name] = file_list
+
+
+def get_heat_templates(config):
+    """
+    Returns the Heat template paths discovered by the pytest parameterizers
+    :param config: pytest config
+    :return: list of heat template paths
+    """
+    meta = DummyMetafunc(config)
+    parametrize_heat_templates(meta)
+    heat_templates = meta.inputs.get("heat_templates", [])
+    if isinstance(heat_templates, list) and len(heat_templates) > 0:
+        heat_templates = heat_templates[0]
+    else:
+        return
+    return heat_templates
+
+
+class FilterBaseOutputs(ABC):
+    """
+    Invoked to remove parameters in an object that appear in the base module.
+    Base output parameters can be passed to incremental modules
+    so they do not need to be defined in a preload.  This method can be
+    invoked on a module to pre-filter the parameters before a preload is
+    created.
+
+    The method should remove the parameters that exist in the base module from
+    both itself and any sub-objects.
+    """
+
+    @abstractmethod
+    def filter_output_params(self, base_outputs):
+        raise NotImplementedError()
+
+
+class IpParam:
+    def __init__(self, ip_addr_param, port):
+        self.param = ip_addr_param or ""
+        self.port = port
+
+    @property
+    def ip_version(self):
+        return 6 if "_v6_" in self.param else 4
+
+    def __hash__(self):
+        return hash(self.param)
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+    def __str__(self):
+        return "{}(v{})".format(self.param, self.ip_version)
+
+    def __repr(self):
+        return str(self)
+
+
+class Network(FilterBaseOutputs):
+    def __init__(self, role, name_param):
+        self.network_role = role
+        self.name_param = name_param
+        self.subnet_params = set()
+
+    def filter_output_params(self, base_outputs):
+        self.subnet_params = remove(self.subnet_params, base_outputs)
+
+    def __hash__(self):
+        return hash(self.network_role)
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+
+class Port(FilterBaseOutputs):
+    def __init__(self, vm, network):
+        self.vm = vm
+        self.network = network
+        self.fixed_ips = []
+        self.floating_ips = []
+        self.uses_dhcp = True
+
+    def add_ips(self, props):
+        props = props.get("properties") or props
+        for fixed_ip in props.get("fixed_ips") or []:
+            if not isinstance(fixed_ip, dict):
+                continue
+            ip_address = get_param(fixed_ip.get("ip_address"))
+            subnet = get_param(fixed_ip.get("subnet") or fixed_ip.get("subnet_id"))
+            if ip_address:
+                self.uses_dhcp = False
+                self.fixed_ips.append(IpParam(ip_address, self))
+            if subnet:
+                self.network.subnet_params.add(subnet)
+        for ip in prop_iterator(props, "allowed_address_pairs", "ip_address"):
+            self.uses_dhcp = False
+            param = get_param(ip) if ip else ""
+            if param:
+                self.floating_ips.append(IpParam(param, self))
+
+    def filter_output_params(self, base_outputs):
+        self.fixed_ips = remove(self.fixed_ips, base_outputs, key=lambda ip: ip.param)
+        self.floating_ips = remove(
+            self.floating_ips, base_outputs, key=lambda ip: ip.param
+        )
+
+
+class VirtualMachineType(FilterBaseOutputs):
+    def __init__(self, vm_type, vnf_module):
+        self.vm_type = vm_type
+        self.names = []
+        self.ports = []
+        self.vm_count = 0
+        self.vnf_module = vnf_module
+
+    def filter_output_params(self, base_outputs):
+        self.names = remove(self.names, base_outputs)
+        for port in self.ports:
+            port.filter_output_params(base_outputs)
+
+    @property
+    def networks(self):
+        return {port.network for port in self.ports}
+
+    @property
+    def floating_ips(self):
+        for port in self.ports:
+            for ip in port.floating_ips:
+                yield ip
+
+    @property
+    def fixed_ips(self):
+        for port in self.ports:
+            for ip in port.fixed_ips:
+                yield ip
+
+    def update_ports(self, network, props):
+        port = self.get_or_create_port(network)
+        port.add_ips(props)
+
+    def get_or_create_port(self, network):
+        for port in self.ports:
+            if port.network == network:
+                return port
+        port = Port(self, network)
+        self.ports.append(port)
+        return port
+
+
+class Vnf:
+    def __init__(self, templates):
+        self.modules = [VnfModule(t, self) for t in templates]
+        self.uses_contrail = self._uses_contrail()
+        self.base_module = next(
+            (mod for mod in self.modules if mod.is_base_module), None
+        )
+        self.incremental_modules = [m for m in self.modules if not m.is_base_module]
+
+    def _uses_contrail(self):
+        for mod in self.modules:
+            resources = mod.heat.get_all_resources()
+            types = (r.get("type", "") for r in resources.values())
+            if any(t.startswith("OS::ContrailV2") for t in types):
+                return True
+        return False
+
+    @property
+    def base_output_params(self):
+        return self.base_module.heat.outputs
+
+    def filter_base_outputs(self):
+        non_base_modules = (m for m in self.modules if not m.is_base_module)
+        for mod in non_base_modules:
+            mod.filter_output_params(self.base_output_params)
+
+
+def env_path(heat_path):
+    """
+    Create the path to the env file for the give heat path.
+    :param heat_path: path to heat file
+    :return: path to env file (assumes it is present and named correctly)
+    """
+    base_path = os.path.splitext(heat_path)[0]
+    return "{}.env".format(base_path)
+
+
+class VnfModule(FilterBaseOutputs):
+    def __init__(self, template_file, vnf):
+        self.vnf = vnf
+        self.vnf_name = os.path.splitext(os.path.basename(template_file))[0]
+        self.template_file = template_file
+        self.heat = Heat(filepath=template_file, envpath=env_path(template_file))
+        env_pair = get_environment_pair(self.template_file)
+        env_yaml = env_pair.get("eyml") if env_pair else {}
+        self.parameters = env_yaml.get("parameters") or {}
+        self.networks = []
+        self.virtual_machine_types = self._create_vm_types()
+        self._add_networks()
+        self.outputs_filtered = False
+
+    def filter_output_params(self, base_outputs):
+        for vm in self.virtual_machine_types:
+            vm.filter_output_params(base_outputs)
+        for network in self.networks:
+            network.filter_output_params(base_outputs)
+        self.parameters = {
+            k: v for k, v in self.parameters.items() if k not in base_outputs
+        }
+        self.networks = [
+            network
+            for network in self.networks
+            if network.name_param not in base_outputs or network.subnet_params
+        ]
+        self.outputs_filtered = True
+
+    def _create_vm_types(self):
+        servers = self.heat.get_resource_by_type("OS::Nova::Server", all_resources=True)
+        vm_types = {}
+        for _, props in yield_by_count(servers):
+            vm_type = get_vm_type_for_nova_server(props)
+            vm = vm_types.setdefault(vm_type, VirtualMachineType(vm_type, self))
+            vm.vm_count += 1
+            name = nested_dict.get(props, "properties", "name", default={})
+            vm_name = get_param(name) if name else ""
+            vm.names.append(vm_name)
+        return list(vm_types.values())
+
+    def _add_networks(self):
+        ports = self.heat.get_resource_by_type("OS::Neutron::Port", all_resources=True)
+        for rid, props in yield_by_count(ports):
+            resource_type, port_match = NeutronPortProcessor.get_rid_match_tuple(rid)
+            if resource_type != "external":
+                continue
+            network_role = port_match.group("network_role")
+            vm = self._get_vm_type(port_match.group("vm_type"))
+            network = self._get_network(network_role, props)
+            vm.update_ports(network, props)
+
+    @property
+    def is_base_module(self):
+        return is_base_module(self.template_file)
+
+    @property
+    def availability_zones(self):
+        """Returns a list of all availability zone parameters found in the template"""
+        return sorted(
+            p for p in self.heat.parameters if p.startswith("availability_zone")
+        )
+
+    @property
+    def label(self):
+        """
+        Label for the VF module that will appear in the CSAR
+        """
+        return self.vnf_name
+
+    @property
+    def env_specs(self):
+        """Return available Environment Spec definitions"""
+        return Config().env_specs
+
+    @property
+    def env_template(self):
+        """
+        Returns a a template .env file that can be completed to enable
+        preload generation.
+        """
+        params = {}
+        params["vnf-name"] = CHANGE
+        params["vnf-type"] = CHANGE
+        params["vf-module-model-name"] = CHANGE
+        params["vf_module_name"] = CHANGE
+        for network in self.networks:
+            params[network.name_param] = CHANGE
+            for param in set(network.subnet_params):
+                params[param] = CHANGE
+        for vm in self.virtual_machine_types:
+            for name in set(vm.names):
+                params[name] = CHANGE
+            for ip in vm.floating_ips:
+                params[ip.param] = CHANGE
+            for ip in vm.fixed_ips:
+                params[ip.param] = CHANGE
+        excluded = get_preload_excluded_parameters(
+            self.template_file, persistent_only=True
+        )
+        for name, value in self.parameters.items():
+            if name in excluded:
+                continue
+            params[name] = value
+        return {"parameters": params}
+
+    @property
+    def preload_parameters(self):
+        """
+        Subset of parameters from the env file that can be overridden in
+        tag values. Per VNF Heat Guidelines, specific parameters such as
+        flavor, image, etc. must not be overridden so they are excluded.
+
+        :return: dict of parameters suitable for the preload
+        """
+        excluded = get_preload_excluded_parameters(self.template_file)
+        return {k: v for k, v in self.parameters.items() if k not in excluded}
+
+    def _get_vm_type(self, vm_type):
+        for vm in self.virtual_machine_types:
+            if vm_type.lower() == vm.vm_type.lower():
+                return vm
+        raise RuntimeError("Encountered unknown VM type: {}".format(vm_type))
+
+    def _get_network(self, network_role, props):
+        network_prop = nested_dict.get(props, "properties", "network") or {}
+        name_param = get_param(network_prop) if network_prop else ""
+        for network in self.networks:
+            if network.network_role.lower() == network_role.lower():
+                return network
+        new_network = Network(network_role, name_param)
+        self.networks.append(new_network)
+        return new_network
+
+    def __str__(self):
+        return "VNF Module ({})".format(os.path.basename(self.template_file))
+
+    def __repr__(self):
+        return str(self)
+
+    def __hash__(self):
+        return hash(self.vnf_name)
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+
+def create_preloads(config, exitstatus):
+    """
+    Create preloads in every format that can be discovered by get_generator_plugins
+    """
+    if config.getoption("self_test"):
+        return
+    print("+===================================================================+")
+    print("|                      Preload Template Generation                  |")
+    print("+===================================================================+")
+
+    preload_dir = os.path.join(get_output_dir(config), "preloads")
+    if os.path.exists(preload_dir):
+        shutil.rmtree(preload_dir)
+    env_directory = config.getoption("env_dir")
+    preload_env = PreloadEnvironment(env_directory) if env_directory else None
+    plugins = get_generator_plugins()
+    available_formats = [p.format_name() for p in plugins]
+    selected_formats = config.getoption("preload_formats") or available_formats
+    heat_templates = get_heat_templates(config)
+    vnf = None
+    for plugin_class in plugins:
+        if plugin_class.format_name() not in selected_formats:
+            continue
+        vnf = Vnf(heat_templates)
+        generator = plugin_class(vnf, preload_dir, preload_env)
+        generator.generate()
+    if vnf and vnf.uses_contrail:
+        print(
+            "\nWARNING: Preload template generation does not support Contrail\n"
+            "at this time, but Contrail resources were detected. The preload \n"
+            "template may be incomplete."
+        )
+    if exitstatus != 0:
+        print(
+            "\nWARNING: Heat violations detected. Preload templates may be\n"
+            "incomplete."
+        )