diff options
Diffstat (limited to 'ice_validator/preload')
| -rw-r--r-- | ice_validator/preload/__init__.py | 4 | ||||
| -rw-r--r-- | ice_validator/preload/data.py | 372 | ||||
| -rw-r--r-- | ice_validator/preload/engine.py | 114 | ||||
| -rw-r--r-- | ice_validator/preload/environment.py | 135 | ||||
| -rw-r--r-- | ice_validator/preload/generator.py | 145 | ||||
| -rw-r--r-- | ice_validator/preload/model.py | 136 |
6 files changed, 765 insertions, 141 deletions
diff --git a/ice_validator/preload/__init__.py b/ice_validator/preload/__init__.py index 70f9ecb..ec6ad7b 100644 --- a/ice_validator/preload/__init__.py +++ b/ice_validator/preload/__init__.py @@ -34,3 +34,7 @@ # limitations under the License. # # ============LICENSE_END============================================ + +from preload.environment import EnvironmentFileDataSource + +__all__ = ["EnvironmentFileDataSource"] diff --git a/ice_validator/preload/data.py b/ice_validator/preload/data.py new file mode 100644 index 0000000..721608f --- /dev/null +++ b/ice_validator/preload/data.py @@ -0,0 +1,372 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Iterable, Any, Optional, Mapping + +from preload.model import VnfModule + + +class AbstractPreloadInstance(ABC): + """ + Represents the data source for a single instance of a preload for + any format. The implementation of AbstractPreloadGenerator will + call the methods of this class to retrieve the necessary data + to populate the preload. If a data element is not available, + then simply return ``None`` and a suitable placeholder will be + placed in the preload. + """ + + @property + @abstractmethod + def output_dir(self) -> Path: + """ + Base output directory where the preload will be generated. Please + note, that the generator may create nested directories under this + directory for the preload. + + :return: Path to the desired output directory. This directory + and its parents will be created by the generator if + it is not already present. + """ + raise NotImplementedError() + + @property + @abstractmethod + def module_label(self) -> str: + """ + Identifier of the module. This must match the base name of the + heat module (ex: if the Heat file name is base.yaml, then the label + is 'base'. + + :return: string name of the module + """ + raise NotImplementedError() + + @property + @abstractmethod + def vf_module_name(self) -> Optional[str]: + """ + :return: module name to populate in the preload if available + """ + raise NotImplementedError() + + @property + @abstractmethod + def flag_incompletes(self) -> bool: + """ + If True, then the generator will modify the file name of any + generated preload to end with _incomplete.<ext> if any preload + value was not satisfied by the data source. If False, then + the file name will be the same regardless of the completeness + of the preload. + + :return: True if file names should denote preload incompleteness + """ + raise NotImplementedError() + + @property + @abstractmethod + def preload_basename(self) -> str: + """ + Base name of the preload that will be used by the generator to create + the file name. + """ + raise NotImplementedError() + + @property + @abstractmethod + def vnf_name(self) -> Optional[str]: + """ + :return: the VNF name to populate in the prelad if available + """ + raise NotImplementedError() + + @property + @abstractmethod + def vnf_type(self) -> Optional[str]: + """ + The VNF Type must be match the values in SDC. It is a concatenation + of <Service Instance Name>/<Resource Instance Name>. + + :return: VNF Type to populate in the preload if available + """ + raise NotImplementedError() + + @property + @abstractmethod + def vf_module_model_name(self) -> Optional[str]: + """ + :return: Module model name if available + """ + raise NotImplementedError() + + @abstractmethod + def get_availability_zone(self, index: int, param_name: str) -> Optional[str]: + """ + Retrieve the value for the availability zone at requested zero-based + index (i.e. 0, 1, 2, etc.) + + :param index: index of availability zone (0, 1, etc.) + :param param_name: Name of the parameter from Heat + :return: value for the AZ if available + """ + raise NotImplementedError() + + @abstractmethod + def get_network_name(self, network_role: str, name_param: str) -> Optional[str]: + """ + Retrieve the OpenStack name of the network for the given network role. + + :param network_role: Network role from Heat template + :param name_param: Network name parameter from Heat + :return: Name of the network if available + """ + raise NotImplementedError() + + @abstractmethod + def get_subnet_id( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + """ + Retrieve the subnet's UUID for the given network and IP version (4 or 6). + + :param network_role: Network role from Heat template + :param ip_version: IP Version (4 or 6) + :param param_name: Parameter name from Heat + :return: UUID of the subnet if available + """ + raise NotImplementedError() + + @abstractmethod + def get_subnet_name( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + """ + Retrieve the OpenStack Subnet name for the given network role and IP version + + :param network_role: Network role from Heat template + :param ip_version: IP Version (4 or 6) + :param param_name: Parameter name from Heat + :return: Name of the subnet if available + """ + raise NotImplementedError() + + @abstractmethod + def get_vm_name(self, vm_type: str, index: int, param_name: str) -> Optional[str]: + """ + Retrieve the vm name for the given VM type and index. + + :param vm_type: VM Type from Heat template + :param index: Zero-based index of the VM for the vm-type + :param param_name: Parameter name from Heat + :return: VM Name if available + """ + raise NotImplementedError() + + @abstractmethod + def get_floating_ip( + self, vm_type: str, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + """ + Retreive the floating IP for the VM and Port identified by VM Type, + Network Role, and IP Version. + + :param vm_type: VM Type from Heat template + :param network_role: Network Role from Heat template + :param ip_version: IP Version (4 or 6) + :param param_name: Parameter name from Heat + :return: floating IP address if available + """ + raise NotImplementedError() + + @abstractmethod + def get_fixed_ip( + self, vm_type: str, network_role: str, ip_version: int, index: int, param: str + ) -> Optional[str]: + """ + Retreive the fixed IP for the VM and Port identified by VM Type, + Network Role, IP Version, and index. + + :param vm_type: VM Type from Heat template + :param network_role: Network Role from Heat template + :param ip_version: IP Version (4 or 6) + :param index: zero-based index for the IP for the given + VM Type, Network Role, IP Version combo + :param param_name: Parameter name from Heat + :return: floating IP address if available + """ + raise NotImplementedError() + + @abstractmethod + def get_vnf_parameter(self, key: str, value: Any) -> Optional[Any]: + """ + Retrieve the value for the given key. These will be placed in the + tag-values/vnf parameters in the preload. If a value was specified in + the environment packaged in the Heat for for the VNF module, then + that value will be passed in ``value``. This class can return + the value or ``None`` if it does not have a value for the given key. + + :param key: parameter name from Heat + :param value: Value from Heat env file if it was assigned there; + None otherwise + :return: Returns the value for the object. This should + be a str, dict, or list. The generator will + format it properly based on the selected output format + """ + raise NotImplementedError() + + @abstractmethod + def get_additional_parameters(self) -> Mapping[str, Any]: + """ + Return any additional parameters that should be added to the VNF parameters. + + This can be useful if you want to duplicate paramters in tag values that are + also in the other sections (ex: VM names). + + :return: dict of str to object mappings that the generator must add to + the vnf_parameters/tag values + """ + raise NotImplementedError() + + +class AbstractPreloadDataSource(ABC): + """ + Represents a data source for a VNF preload data. Implementations of this + class can be dynamically discovered if they are in a preload plugin module. + A module is considered a preload plugin module if it starts with + prelaod_ and is available as a top level module on Python's sys.path. + + The ``get_module_preloads`` will be invoked for each module in + the VNF. An instance of AbstractPreloadInstance must be returned for + each instance of the preload module that is to be created. + + Parameters: + :param path: The path to the configuration source selected + in either the VVP GUI or command-line. This + may be a file or directory depending upon + the source_type defined by this data source + """ + + def __init__(self, path: Path): + self.path = path + + @classmethod + @abstractmethod + def get_source_type(cls) -> str: + """ + If 'FILE' returned, then the config source will be a specific + file; If 'DIR', then the config source will be a directory + :return: + """ + raise NotImplementedError() + + @classmethod + @abstractmethod + def get_identifier(cls) -> str: + """ + Identifier for the given data source. This is the value that + can be passed via --preload-source-type. + + :return: short identifier for this data source type + """ + + @classmethod + @abstractmethod + def get_name(self) -> str: + """ + Human readable name to describe the preload data source. It is + recommended not to exceed 50 characters. + + :return: human readable name of the preload data source (ex: Environment Files) + """ + raise NotImplementedError() + + @abstractmethod + def get_module_preloads( + self, module: VnfModule + ) -> Iterable[AbstractPreloadInstance]: + """ + For the requested module, return an instance of AbstractPreloadInstance + for every preload module you wish to be created. + + :param module: Module of the VNF + :return: iterable of preloads to create for the given module + """ + raise NotImplementedError() + + +class BlankPreloadInstance(AbstractPreloadInstance): + """ + Used to create blank preload templates. VVP will always create + a template of a preload in the requested format with no data provided. + """ + + def __init__(self, output_dir: Path, module_name: str): + self._output_dir = output_dir + self._module_name = module_name + + @property + def flag_incompletes(self) -> bool: + return False + + @property + def preload_basename(self) -> str: + return self._module_name + + @property + def vf_module_name(self) -> Optional[str]: + return None + + def get_vm_name(self, vm_type: str, index: int, param_name: str) -> Optional[str]: + return None + + def get_availability_zone(self, index: int, param_name: str) -> Optional[str]: + return None + + @property + def output_dir(self) -> Path: + return self._output_dir + + @property + def module_label(self) -> str: + return self._module_name + + @property + def vnf_name(self) -> Optional[str]: + return None + + @property + def vnf_type(self) -> Optional[str]: + return None + + @property + def vf_module_model_name(self) -> Optional[str]: + return None + + def get_network_name(self, network_role: str, name_param: str) -> Optional[str]: + return None + + def get_subnet_id( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + return None + + def get_subnet_name( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + return None + + def get_floating_ip( + self, vm_type: str, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + return None + + def get_fixed_ip( + self, vm_type: str, network_role: str, ip_version: int, index: int, param: str + ) -> Optional[str]: + return None + + def get_vnf_parameter(self, key: str, value: Any) -> Optional[Any]: + return None + + def get_additional_parameters(self) -> Mapping[str, Any]: + return {} diff --git a/ice_validator/preload/engine.py b/ice_validator/preload/engine.py new file mode 100644 index 0000000..488766d --- /dev/null +++ b/ice_validator/preload/engine.py @@ -0,0 +1,114 @@ +import importlib +import inspect +import os +import pkgutil +import shutil +from itertools import chain +from pathlib import Path +from typing import List, Type + +from preload.data import AbstractPreloadDataSource +from preload.generator import AbstractPreloadGenerator +from preload.model import get_heat_templates, Vnf +from tests.helpers import get_output_dir + + +def create_preloads(config, exitstatus): + """ + Create preloads in every format that can be discovered by get_generator_plugins + """ + if config.getoption("self_test"): + return + print("+===================================================================+") + print("| Preload Template Generation |") + print("+===================================================================+") + + preload_dir = os.path.join(get_output_dir(config), "preloads") + if os.path.exists(preload_dir): + shutil.rmtree(preload_dir) + plugins = PluginManager() + available_formats = [p.format_name() for p in plugins.preload_generators] + selected_formats = config.getoption("preload_formats") or available_formats + preload_source = None + if config.getoption("preload_source"): + preload_source_path = Path(config.getoption("preload_source")) + source_class = plugins.get_source_for_id( + config.getoption("preload_source_type") + ) + preload_source = source_class(preload_source_path) + + heat_templates = get_heat_templates(config) + vnf = None + for plugin_class in plugins.preload_generators: + if plugin_class.format_name() not in selected_formats: + continue + vnf = Vnf(heat_templates) + generator = plugin_class(vnf, preload_dir, preload_source) + generator.generate() + if vnf and vnf.uses_contrail: + print( + "\nWARNING: Preload template generation does not support Contrail\n" + "at this time, but Contrail resources were detected. The preload \n" + "template may be incomplete." + ) + if exitstatus != 0: + print( + "\nWARNING: Heat violations detected. Preload templates may be\n" + "incomplete or have errors." + ) + + +def is_implementation_of(class_, base_class): + """ + Returns True if the class is an implementation of AbstractPreloadGenerator + """ + return ( + inspect.isclass(class_) + and not inspect.isabstract(class_) + and issubclass(class_, base_class) + ) + + +def get_implementations_of(class_, modules): + """ + Returns all classes that implement ``class_`` from modules + """ + members = list( + chain.from_iterable( + inspect.getmembers(mod, lambda c: is_implementation_of(c, class_)) + for mod in modules + ) + ) + return [m[1] for m in members] + + +class PluginManager: + def __init__(self): + self.preload_plugins = [ + importlib.import_module(name) + for finder, name, ispkg in pkgutil.iter_modules() + if name.startswith("preload_") or name == "preload" + ] + self.preload_generators: List[ + Type[AbstractPreloadGenerator] + ] = get_implementations_of(AbstractPreloadGenerator, self.preload_plugins) + self.preload_sources: List[ + Type[AbstractPreloadDataSource] + ] = get_implementations_of(AbstractPreloadDataSource, self.preload_plugins) + + def get_source_for_id(self, identifier: str) -> Type[AbstractPreloadDataSource]: + for source in self.preload_sources: + if identifier == source.get_identifier(): + return source + raise RuntimeError( + "Unable to find preload source for identifier {}".format(identifier) + ) + + def get_source_for_name(self, name: str) -> Type[AbstractPreloadDataSource]: + for source in self.preload_sources: + if name == source.get_name(): + return source + raise RuntimeError("Unable to find preload source for name {}".format(name)) + + +PLUGIN_MGR = PluginManager() diff --git a/ice_validator/preload/environment.py b/ice_validator/preload/environment.py index 083be9b..0477e66 100644 --- a/ice_validator/preload/environment.py +++ b/ice_validator/preload/environment.py @@ -1,14 +1,19 @@ import re import tempfile from pathlib import Path +from typing import Any, Optional, Mapping from cached_property import cached_property +from preload.data import AbstractPreloadInstance, AbstractPreloadDataSource +from preload.model import VnfModule from tests.helpers import check, first, unzip, load_yaml SERVICE_TEMPLATE_PATTERN = re.compile(r".*service-.*?-template.yml") RESOURCE_TEMPLATE_PATTERN = re.compile(r".*resource-(.*?)-template.yml") +ZONE_PARAMS = ("availability_zone_0", "availability_zone_1", "availability_zone_2") + def yaml_files(path): """ @@ -278,3 +283,133 @@ class PreloadEnvironment: def __repr__(self): return "PreloadEnvironment(name={})".format(self.name) + + +class EnvironmentFilePreloadInstance(AbstractPreloadInstance): + + def __init__(self, env: PreloadEnvironment, module_label: str, module_params: dict): + self.module_params = module_params + self._module_label = module_label + self.env = env + self.env_cache = {} + + @property + def flag_incompletes(self) -> bool: + return True + + @property + def preload_basename(self) -> str: + return self.module_label + + @property + def output_dir(self) -> Path: + return self.env.base_dir.joinpath("preloads") + + @property + def module_label(self) -> str: + return self._module_label + + @property + def vf_module_name(self) -> str: + return self.get_param("vf_module_name") + + @property + def vnf_name(self) -> Optional[str]: + return self.get_param("vnf_name") + + @property + def vnf_type(self) -> Optional[str]: + return self.get_param("vnf-type") + + @property + def vf_module_model_name(self) -> Optional[str]: + return self.get_param("vf-module-model-name") + + def get_availability_zone(self, index: int, param_name: str) -> Optional[str]: + return self.get_param(param_name) + + def get_network_name(self, network_role: str, name_param: str) -> Optional[str]: + return self.get_param(name_param) + + def get_subnet_id( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + return self.get_param(param_name) + + def get_subnet_name( + self, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + # Not supported with env files + return None + + def get_vm_name(self, vm_type: str, index: int, param_name: str) -> Optional[str]: + return self.get_param(param_name, single=True) + + def get_floating_ip( + self, vm_type: str, network_role: str, ip_version: int, param_name: str + ) -> Optional[str]: + return self.get_param(param_name) + + def get_fixed_ip( + self, vm_type: str, network_role: str, ip_version: int, index: int, param: str + ) -> Optional[str]: + return self.get_param(param, single=True) + + def get_vnf_parameter(self, key: str, value: Any) -> Optional[str]: + module_value = self.get_param(key) + return module_value or value + + def get_additional_parameters(self) -> Mapping[str, Any]: + return {} + + def get_param(self, param_name, single=False): + """ + Retrieves the value for the given param if it exists. If requesting a + single item, and the parameter is tied to a list then only one item from + the list will be returned. For each subsequent call with the same parameter + it will iterate/rotate through the values in that list. If single is False + then the full list will be returned. + + :param param_name: name of the parameter + :param single: If True returns single value from lists otherwises the full + list. This has no effect on non-list values + """ + value = self.env_cache.get(param_name) + if not value: + value = self.module_params.get(param_name) + if isinstance(value, list): + value = value.copy() + value.reverse() + self.env_cache[param_name] = value + + if value and single and isinstance(value, list): + result = value.pop() + else: + result = value + return result if result != "CHANGEME" else None + + +class EnvironmentFileDataSource(AbstractPreloadDataSource): + + def __init__(self, path: Path): + super().__init__(path) + check(path.is_dir(), f"{path} must be an existing directory") + self.path = path + self.env = PreloadEnvironment(path) + + @classmethod + def get_source_type(cls) -> str: + return "DIR" + + @classmethod + def get_identifier(self) -> str: + return "envfiles" + + @classmethod + def get_name(self) -> str: + return "Environment Files" + + def get_module_preloads(self, module: VnfModule): + for env in self.env.environments: + module_params = env.get_module(module.label) + yield EnvironmentFilePreloadInstance(env, module.label, module_params) diff --git a/ice_validator/preload/generator.py b/ice_validator/preload/generator.py index bdd81fa..ffdc420 100644 --- a/ice_validator/preload/generator.py +++ b/ice_validator/preload/generator.py @@ -39,9 +39,17 @@ import json import os from abc import ABC, abstractmethod from collections import OrderedDict +from pathlib import Path import yaml +from preload.data import ( + AbstractPreloadDataSource, + AbstractPreloadInstance, + BlankPreloadInstance, +) +from preload.model import VnfModule, Vnf + def represent_ordered_dict(dumper, data): value = [] @@ -76,26 +84,15 @@ def get_or_create_template(template_dir, key, value, sequence, template_name): return new_template -def yield_by_count(sequence): - """ - Iterates through sequence and yields each item according to its __count__ - attribute. If an item has a __count__ of it will be returned 3 times - before advancing to the next item in the sequence. - - :param sequence: sequence of dicts (must contain __count__) - :returns: generator of tuple key, value pairs - """ - for key, value in sequence.items(): - for i in range(value["__count__"]): - yield (key, value) - - -def replace(param): +def replace(param, index=None): """ Optionally used by the preload generator to wrap items in the preload that need to be replaced by end users - :param param: p + :param param: parameter name + :param index: optional index (int or str) of the parameter """ + if (param.endswith("_names") or param.endswith("_ips")) and index is not None: + param = "{}[{}]".format(param, index) return "VALUE FOR: {}".format(param) if param else "" @@ -113,15 +110,15 @@ class AbstractPreloadGenerator(ABC): :param vnf: Instance of Vnf that contains the preload data :param base_output_dir: Base directory to house the preloads. All preloads must be written to a subdirectory under this directory + :param data_source: Source data for preload population """ - def __init__(self, vnf, base_output_dir, preload_env): - self.preload_env = preload_env + def __init__( + self, vnf: Vnf, base_output_dir: Path, data_source: AbstractPreloadDataSource + ): + self.data_source = data_source self.vnf = vnf - self.current_module = None - self.current_module_env = {} self.base_output_dir = base_output_dir - self.env_cache = {} self.module_incomplete = False @classmethod @@ -158,11 +155,10 @@ class AbstractPreloadGenerator(ABC): raise NotImplementedError() @abstractmethod - def generate_module(self, module, output_dir): + def generate_module(self, module: VnfModule, preload: AbstractPreloadInstance, output_dir: Path): """ - Create the preloads and write them to ``output_dir``. This - method is responsible for generating the content of the preload and - writing the file to disk. + Create the preloads. This method is responsible for generating the + content of the preload and writing the file to disk. """ raise NotImplementedError() @@ -170,29 +166,17 @@ class AbstractPreloadGenerator(ABC): # handle the base module first print("\nGenerating {} preloads".format(self.format_name())) if self.vnf.base_module: - self.generate_environments(self.vnf.base_module) + self.generate_preloads(self.vnf.base_module) if self.supports_output_passing(): self.vnf.filter_base_outputs() for mod in self.vnf.incremental_modules: - self.generate_environments(mod) + self.generate_preloads(mod) - def replace(self, param_name, alt_message=None, single=False): - value = self.get_param(param_name, single) - value = None if value == "CHANGEME" else value - if value: - return value - else: - self.module_incomplete = True - return alt_message or replace(param_name) - - def start_module(self, module, env): + def start_module(self): """Initialize/reset the environment for the module""" - self.current_module = module - self.current_module_env = env self.module_incomplete = False - self.env_cache = {} - def generate_environments(self, module): + def generate_preloads(self, module): """ Generate a preload for the given module in all available environments in the ``self.preload_env``. This will invoke the abstract @@ -204,65 +188,50 @@ class AbstractPreloadGenerator(ABC): print("\nGenerating Preloads for {}".format(module)) print("-" * 50) print("... generating blank template") - self.start_module(module, {}) - blank_preload_dir = self.make_preload_dir(self.base_output_dir) - self.generate_module(module, blank_preload_dir) - self.generate_preload_env(module, blank_preload_dir) - if self.preload_env: - for env in self.preload_env.environments: - output_dir = self.make_preload_dir(env.base_dir / "preloads") + self.start_module() + preload = BlankPreloadInstance(Path(self.base_output_dir), module.label) + blank_preload_dir = self.make_preload_dir(preload) + self.generate_module(module, preload, blank_preload_dir) + self.generate_preload_env(module, preload) + + if self.data_source: + preloads = self.data_source.get_module_preloads(module) + for preload in preloads: + output_dir = self.make_preload_dir(preload) print( - "... generating preload for env ({}) to {}".format( - env.name, output_dir + "... generating preload for {} to {}".format( + preload.module_label, output_dir ) ) - self.start_module(module, env.get_module(module.label)) - self.generate_module(module, output_dir) + self.start_module() + self.generate_module(module, preload, output_dir) - def make_preload_dir(self, base_dir): - path = os.path.join(base_dir, self.output_sub_dir()) - if not os.path.exists(path): - os.makedirs(path, exist_ok=True) - return path + def make_preload_dir(self, preload: AbstractPreloadInstance): + preload_dir = preload.output_dir.joinpath(self.output_sub_dir()) + preload_dir.mkdir(parents=True, exist_ok=True) + return preload_dir @staticmethod - def generate_preload_env(module, blank_preload_dir): + def generate_preload_env(module: VnfModule, preload: AbstractPreloadInstance): """ Create a .env template suitable for completing and using for preload generation from env files. """ yaml.add_representer(OrderedDict, represent_ordered_dict) - output_dir = os.path.join(blank_preload_dir, "preload_env") - env_file = os.path.join(output_dir, "{}.env".format(module.vnf_name)) - defaults_file = os.path.join(output_dir, "defaults.yaml") - if not os.path.exists(output_dir): - os.makedirs(output_dir, exist_ok=True) - with open(env_file, "w") as f: + output_dir = preload.output_dir.joinpath("preload_env") + env_file = output_dir.joinpath("{}.env".format(module.label)) + defaults_file = output_dir.joinpath("defaults.yaml") + output_dir.mkdir(parents=True, exist_ok=True) + with env_file.open("w") as f: yaml.dump(module.env_template, f) - if not os.path.exists(defaults_file): - with open(defaults_file, "w") as f: + if not defaults_file.exists(): + with defaults_file.open("w") as f: yaml.dump({"vnf_name": "CHANGEME"}, f) - def get_param(self, param_name, single): - """ - Retrieves the value for the given param if it exists. If requesting a - single item, and the parameter is tied to a list then only one item from - the list will be returned. For each subsequent call with the same parameter - it will iterate/rotate through the values in that list. If single is False - then the full list will be returned. - - :param param_name: name of the parameter - :param single: If True returns single value from lists otherwises the full - list. This has no effect on non-list values - """ - value = self.env_cache.get(param_name) - if not value: - value = self.current_module_env.get(param_name) - if isinstance(value, list): - value = value.copy() - value.reverse() - self.env_cache[param_name] = value - if value and single and isinstance(value, list): - return value.pop() + def normalize(self, preload_value, param_name, alt_message=None, index=None): + preload_value = None if preload_value == "CHANGEME" else preload_value + if preload_value: + return preload_value else: - return value + self.module_incomplete = True + return alt_message or replace(param_name, index) diff --git a/ice_validator/preload/model.py b/ice_validator/preload/model.py index 3ca7bda..21d849e 100644 --- a/ice_validator/preload/model.py +++ b/ice_validator/preload/model.py @@ -35,17 +35,15 @@ # # ============LICENSE_END============================================ import os -import shutil from abc import ABC, abstractmethod from collections import OrderedDict +from itertools import chain +from typing import Tuple, List -from preload.generator import yield_by_count -from preload.environment import PreloadEnvironment from tests.helpers import ( get_param, get_environment_pair, prop_iterator, - get_output_dir, is_base_module, remove, ) @@ -54,7 +52,6 @@ from tests.structures import NeutronPortProcessor, Heat from tests.test_environment_file_parameters import get_preload_excluded_parameters from tests.utils import nested_dict from tests.utils.vm_types import get_vm_type_for_nova_server -from config import Config, get_generator_plugins from tests.test_environment_file_parameters import ENV_PARAMETER_SPEC @@ -133,7 +130,9 @@ class Network(FilterBaseOutputs): self.subnet_params = set() def filter_output_params(self, base_outputs): - self.subnet_params = remove(self.subnet_params, base_outputs) + self.subnet_params = remove( + self.subnet_params, base_outputs, key=lambda s: s.param_name + ) def __hash__(self): return hash(self.network_role) @@ -142,12 +141,27 @@ class Network(FilterBaseOutputs): return hash(self) == hash(other) +class Subnet: + def __init__(self, param_name: str): + self.param_name = param_name + + @property + def ip_version(self): + return 6 if "_v6_" in self.param_name else 4 + + def __hash__(self): + return hash(self.param_name) + + def __eq__(self, other): + return hash(self) == hash(other) + + class Port(FilterBaseOutputs): def __init__(self, vm, network): self.vm = vm self.network = network self.fixed_ips = [] - self.floating_ips = [] + self.floating_ips = set() self.uses_dhcp = True def add_ips(self, props): @@ -161,12 +175,35 @@ class Port(FilterBaseOutputs): self.uses_dhcp = False self.fixed_ips.append(IpParam(ip_address, self)) if subnet: - self.network.subnet_params.add(subnet) + self.network.subnet_params.add(Subnet(subnet)) for ip in prop_iterator(props, "allowed_address_pairs", "ip_address"): - self.uses_dhcp = False param = get_param(ip) if ip else "" if param: - self.floating_ips.append(IpParam(param, self)) + self.floating_ips.add(IpParam(param, self)) + + @property + def ipv6_fixed_ips(self): + return list( + sorted( + (ip for ip in self.fixed_ips if ip.ip_version == 6), + key=lambda ip: ip.param, + ) + ) + + @property + def ipv4_fixed_ips(self): + return list( + sorted( + (ip for ip in self.fixed_ips if ip.ip_version == 4), + key=lambda ip: ip.param, + ) + ) + + @property + def fixed_ips_with_index(self) -> List[Tuple[int, IpParam]]: + ipv4s = enumerate(self.ipv4_fixed_ips) + ipv6s = enumerate(self.ipv6_fixed_ips) + return list(chain(ipv4s, ipv6s)) def filter_output_params(self, base_outputs): self.fixed_ips = remove(self.fixed_ips, base_outputs, key=lambda ip: ip.param) @@ -218,9 +255,10 @@ class VirtualMachineType(FilterBaseOutputs): class Vnf: - def __init__(self, templates): - self.modules = [VnfModule(t, self) for t in templates] + def __init__(self, templates, config=None): + self.modules = [VnfModule(t, self, config) for t in templates] self.uses_contrail = self._uses_contrail() + self.config = config self.base_module = next( (mod for mod in self.modules if mod.is_base_module), None ) @@ -256,8 +294,9 @@ def env_path(heat_path): class VnfModule(FilterBaseOutputs): - def __init__(self, template_file, vnf): + def __init__(self, template_file, vnf, config): self.vnf = vnf + self.config = config self.vnf_name = os.path.splitext(os.path.basename(template_file))[0] self.template_file = template_file self.heat = Heat(filepath=template_file, envpath=env_path(template_file)) @@ -265,11 +304,30 @@ class VnfModule(FilterBaseOutputs): env_yaml = env_pair.get("eyml") if env_pair else {} self.parameters = {key: "" for key in self.heat.parameters} self.parameters.update(env_yaml.get("parameters") or {}) + # Filter out any parameters passed from the volume module's outputs + self.parameters = { + key: value + for key, value in self.parameters.items() + if key not in self.volume_module_outputs + } self.networks = [] self.virtual_machine_types = self._create_vm_types() self._add_networks() self.outputs_filtered = False + @property + def volume_module_outputs(self): + heat_dir = os.path.dirname(self.template_file) + heat_filename = os.path.basename(self.template_file) + basename, ext = os.path.splitext(heat_filename) + volume_template_name = "{}_volume{}".format(basename, ext) + volume_path = os.path.join(heat_dir, volume_template_name) + if os.path.exists(volume_path): + volume_mod = Heat(filepath=volume_path) + return volume_mod.outputs + else: + return {} + def filter_output_params(self, base_outputs): for vm in self.virtual_machine_types: vm.filter_output_params(base_outputs) @@ -329,10 +387,7 @@ class VnfModule(FilterBaseOutputs): @property def env_specs(self): """Return available Environment Spec definitions""" - try: - return Config().env_specs - except FileNotFoundError: - return [ENV_PARAMETER_SPEC] + return [ENV_PARAMETER_SPEC] if not self.config else self.config.env_specs @property def platform_provided_params(self): @@ -356,7 +411,7 @@ class VnfModule(FilterBaseOutputs): params[az] = CHANGE for network in self.networks: params[network.name_param] = CHANGE - for param in set(network.subnet_params): + for param in set(s.param_name for s in network.subnet_params): params[param] = CHANGE for vm in self.virtual_machine_types: for name in set(vm.names): @@ -417,40 +472,15 @@ class VnfModule(FilterBaseOutputs): return hash(self) == hash(other) -def create_preloads(config, exitstatus): +def yield_by_count(sequence): """ - Create preloads in every format that can be discovered by get_generator_plugins + Iterates through sequence and yields each item according to its __count__ + attribute. If an item has a __count__ of it will be returned 3 times + before advancing to the next item in the sequence. + + :param sequence: sequence of dicts (must contain __count__) + :returns: generator of tuple key, value pairs """ - if config.getoption("self_test"): - return - print("+===================================================================+") - print("| Preload Template Generation |") - print("+===================================================================+") - - preload_dir = os.path.join(get_output_dir(config), "preloads") - if os.path.exists(preload_dir): - shutil.rmtree(preload_dir) - env_directory = config.getoption("env_dir") - preload_env = PreloadEnvironment(env_directory) if env_directory else None - plugins = get_generator_plugins() - available_formats = [p.format_name() for p in plugins] - selected_formats = config.getoption("preload_formats") or available_formats - heat_templates = get_heat_templates(config) - vnf = None - for plugin_class in plugins: - if plugin_class.format_name() not in selected_formats: - continue - vnf = Vnf(heat_templates) - generator = plugin_class(vnf, preload_dir, preload_env) - generator.generate() - if vnf and vnf.uses_contrail: - print( - "\nWARNING: Preload template generation does not support Contrail\n" - "at this time, but Contrail resources were detected. The preload \n" - "template may be incomplete." - ) - if exitstatus != 0: - print( - "\nWARNING: Heat violations detected. Preload templates may be\n" - "incomplete." - ) + for key, value in sequence.items(): + for i in range(value["__count__"]): + yield (key, value) |