diff options
Diffstat (limited to 'ice_validator/tests/utils/nested_files.py')
| -rw-r--r-- | ice_validator/tests/utils/nested_files.py | 239 |
1 files changed, 180 insertions, 59 deletions
diff --git a/ice_validator/tests/utils/nested_files.py b/ice_validator/tests/utils/nested_files.py index c551646..aff5a6b 100644 --- a/ice_validator/tests/utils/nested_files.py +++ b/ice_validator/tests/utils/nested_files.py @@ -44,16 +44,77 @@ from os import path import re from tests import cached_yaml as yaml +from tests.structures import Heat -VERSION = '1.0.2' +VERSION = "1.4.0" + +""" +test nesting depth +0 -> 1 -> 2 -> too deep. +""" +MAX_DEPTH = 3 + + +def check_for_invalid_nesting( # pylint: disable=too-many-branches + yml, yaml_file, dirpath +): + """ + return a list of all nested files + """ + if not hasattr(yml, "items"): + return [] + invalid_nesting = [] + p = re.compile("^[A-z]*::[A-z]*::[A-z]*$") + + for v in yml.values(): + if isinstance(v, dict) and "type" in v: + t = v["type"] + if t.endswith(".yml") or t.endswith(".yaml"): + filepath = path.join(dirpath, t) + elif t == "OS::Heat::ResourceGroup": + rd = v["properties"]["resource_def"] + if not isinstance(rd, dict) or "type" not in rd: + invalid_nesting.append(yaml_file) + continue + elif not p.match(rd["type"]): + filepath = path.join(dirpath, rd["type"]) + else: + continue + else: + continue + try: + with open(filepath) as fh: + yml = yaml.load(fh) + except yaml.YAMLError as e: + invalid_nesting.append(filepath) + print(e) # pylint: disable=superfluous-parens + invalid_nesting.extend(check_for_invalid_nesting(yml, filepath, dirpath)) + if isinstance(v, dict): + invalid_nesting.extend(check_for_invalid_nesting(v, yaml_file, dirpath)) + elif isinstance(v, list): + for d in v: + invalid_nesting.extend(check_for_invalid_nesting(d, yaml_file, dirpath)) + return invalid_nesting + + +def get_dict_of_nested_files(yml, dirpath): + """Return dict. + key: resource id in yml which references a nested file. + value: the nested file name. + Nested files are either referenced through "type", or + for OS::Heat::ResourceGroup, through "resource_def type". + """ + nested_files = get_type_nested_files(yml, dirpath) + nested_files.update(get_resourcegroup_nested_files(yml, dirpath)) + return nested_files def get_list_of_nested_files(yml, dirpath): - ''' + """ return a list of all nested files - ''' + """ - if not hasattr(yml, 'items'): + if not hasattr(yml, "items"): return [] nested_files = [] @@ -69,72 +130,132 @@ def get_list_of_nested_files(yml, dirpath): nested_files.append(filepath) nested_files.extend(get_list_of_nested_files(t_yml, dirpath)) elif t == "OS::Heat::ResourceGroup": - rdt = (v.get("properties", {}) - .get("resource_def", {}) - .get("type", None)) + rdt = v.get("properties", {}).get("resource_def", {}).get("type", None) if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")): filepath = path.join(dirpath, rdt) if path.exists(filepath): with open(filepath) as fh: rdt_yml = yaml.load(fh) nested_files.append(filepath) - nested_files.extend( - get_list_of_nested_files(rdt_yml, dirpath)) + nested_files.extend(get_list_of_nested_files(rdt_yml, dirpath)) if isinstance(v, dict): - nested_files.extend( - get_list_of_nested_files(v, dirpath)) + nested_files.extend(get_list_of_nested_files(v, dirpath)) elif isinstance(v, list): for d in v: - nested_files.extend( - get_list_of_nested_files(d, dirpath)) + nested_files.extend(get_list_of_nested_files(d, dirpath)) return nested_files -def check_for_invalid_nesting(yml, yaml_file, dirpath): - ''' - return a list of all nested files - ''' - if not hasattr(yml, 'items'): - return [] - invalid_nesting = [] - p = re.compile('^[A-z]*::[A-z]*::[A-z]*$') +def get_nesting(yaml_files): + """return bad, files, heat, depths + bad - list of error messages. + files - dict: key is filename, value is dict of nested files. + This is the tree. + heat - dict,: key is filename, value is Heat instance. + depths - dict: key is filename, value is a depth tuple - for v in yml.values(): - if isinstance(v, dict) and "type" in v: - t = v["type"] - if t.endswith(".yml") or t.endswith(".yaml"): - filepath = path.join(dirpath, t) - elif t == "OS::Heat::ResourceGroup": - rd = v["properties"]["resource_def"] - if not isinstance(rd, dict) or "type" not in rd: - invalid_nesting.append(yaml_file) - continue - elif not p.match(rd["type"]): - filepath = path.join(dirpath, rd["type"]) - else: - continue - else: - continue - try: - with open(filepath) as fh: - yml = yaml.load(fh) - except yaml.YAMLError as e: - invalid_nesting.append(filepath) - print(e) # pylint: disable=superfluous-parens - invalid_nesting.extend(check_for_invalid_nesting( - yml, - filepath, - dirpath)) - if isinstance(v, dict): - invalid_nesting.extend(check_for_invalid_nesting( - v, - yaml_file, - dirpath)) - elif isinstance(v, list): - for d in v: - invalid_nesting.extend(check_for_invalid_nesting( - d, - yaml_file, - dirpath)) - return invalid_nesting + level: 0 1 2 3 + file: template -> nested -> nested -> nested + depth: 3 2 1 0 + """ + bad = [] + files = {} + heat = {} + depths = {} + for yaml_file in yaml_files: + dirname, basename = path.split(yaml_file) + h = Heat(filepath=yaml_file) + heat[basename] = h + files[basename] = get_dict_of_nested_files(h.yml, dirname) + for filename in files: + depths[filename] = _get_nesting_depth_start(0, filename, files, []) + for depth in depths[filename]: + if depth[0] > MAX_DEPTH: + bad.append("{} {}".format(filename, str(depth[1]))) + return bad, files, heat, depths + + +def _get_nesting_depth_start(depth, filename, files, context): + depths = [] + for rid, nf in files[filename].items(): + depths.append(_get_nesting_depth(1, nf, files, context)) + return depths + + +def _get_nesting_depth(depth, filename, files, context): + """Return a depth tuple (max_depth, current_context). + `context` is the list of filenames. + `depth` is the length of `context`. + Finds the max_depth of all the resources of `filename`. + current_context is the updated list of filenames + and max_depth is its length. + """ + max_depth = depth + 1 + current_context = context + [filename] + if depth <= MAX_DEPTH: + nested_filenames = files.get(filename, {}) + if nested_filenames: + max_depth, current_context = max( + _get_nesting_depth(depth + 1, nested_filename, files, current_context) + for nested_filename in nested_filenames.values() + ) + return max_depth, current_context + +def get_resourcegroup_nested_files(yml, dirpath): + """ + return a dict. + key: key in yml which references a nested ResourceGroup file. + (resource->type is ResourceGroup + and resource->properties->resource_def->type is a yaml file) + value: the nested file name. + + The keys are assumed to be unique across files. + A separate test checks for that. + """ + + if not hasattr(yml, "get"): + return {} + + nested_files = {} + for rid, r in yml.get("resources", {}).items(): + if isinstance(r, dict) and "type" in r: + t = r["type"] + nested_file = None + if t == "OS::Heat::ResourceGroup": + rdt = r.get("properties", {}).get("resource_def", {}).get("type", None) + if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")): + nested_file = rdt + if nested_file: + filepath = path.join(dirpath, nested_file) + if path.exists(filepath): + nested_files[rid] = nested_file + return nested_files + + +def get_type_nested_files(yml, dirpath): + """ + return a dict. + key: key in yml which references a nested type file. + (the resource "type" is a yaml file.) + value: the nested file name. + + The keys are assumed to be unique across files. + A separate test checks for that. + """ + + if not hasattr(yml, "get"): + return {} + + nested_files = {} + for rid, r in yml.get("resources", {}).items(): + if isinstance(r, dict) and "type" in r: + t = r["type"] + nested_file = None + if t.endswith(".yml") or t.endswith(".yaml"): + nested_file = t + if nested_file: + filepath = path.join(dirpath, nested_file) + if path.exists(filepath): + nested_files[rid] = nested_file + return nested_files |