# -*- coding: utf8 -*-
# ============LICENSE_START====================================================
# org.onap.vvp/validation-scripts
# ===================================================================
# Copyright © 2017 AT&T Intellectual Property. All rights reserved.
# ===================================================================
#
# Unless otherwise specified, all software contained herein is licensed
# under the Apache License, Version 2.0 (the "License");
# you may not use this software except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
#
# Unless otherwise specified, all documentation contained herein is licensed
# under the Creative Commons License, Attribution 4.0 Intl. (the "License");
# you may not use this documentation except in compliance with the License.
# You may obtain a copy of the License at
#
# https://creativecommons.org/licenses/by/4.0/
#
# Unless required by applicable law or agreed to in writing, documentation
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ============LICENSE_END============================================
#
#
"""nested files
"""
from os import path
import re
from tests import cached_yaml as yaml
from tests.structures import Heat
VERSION = "1.4.0"
"""
test nesting depth
0 -> 1 -> 2 -> too deep.
"""
MAX_DEPTH = 3
def check_for_invalid_nesting( # pylint: disable=too-many-branches
yml, yaml_file, dirpath
):
"""
return a list of all nested files
"""
if not hasattr(yml, "items"):
return []
invalid_nesting = []
p = re.compile("^[A-z]*::[A-z]*::[A-z]*$")
for v in yml.values():
if isinstance(v, dict) and "type" in v:
t = v["type"]
if t.endswith(".yml") or t.endswith(".yaml"):
filepath = path.join(dirpath, t)
elif t == "OS::Heat::ResourceGroup":
rd = v["properties"]["resource_def"]
if not isinstance(rd, dict) or "type" not in rd:
invalid_nesting.append(yaml_file)
continue
elif not p.match(rd["type"]):
filepath = path.join(dirpath, rd["type"])
else:
continue
else:
continue
try:
with open(filepath) as fh:
yml = yaml.load(fh)
except yaml.YAMLError as e:
invalid_nesting.append(filepath)
print(e) # pylint: disable=superfluous-parens
invalid_nesting.extend(check_for_invalid_nesting(yml, filepath, dirpath))
if isinstance(v, dict):
invalid_nesting.extend(check_for_invalid_nesting(v, yaml_file, dirpath))
elif isinstance(v, list):
for d in v:
invalid_nesting.extend(check_for_invalid_nesting(d, yaml_file, dirpath))
return invalid_nesting
def get_dict_of_nested_files(yml, dirpath):
"""Return dict.
key: resource id in yml which references a nested file.
value: the nested file name.
Nested files are either referenced through "type", or
for OS::Heat::ResourceGroup, through "resource_def type".
"""
nested_files = get_type_nested_files(yml, dirpath)
nested_files.update(get_resourcegroup_nested_files(yml, dirpath))
return nested_files
def get_list_of_nested_files(yml, dirpath):
"""
return a list of all nested files
"""
if not hasattr(yml, "items"):
return []
nested_files = []
for v in yml.values():
if isinstance(v, dict) and "type" in v:
t = v["type"]
if t.endswith(".yml") or t.endswith(".yaml"):
filepath = path.join(dirpath, t)
if path.exists(filepath):
with open(filepath) as fh:
t_yml = yaml.load(fh)
nested_files.append(filepath)
nested_files.extend(get_list_of_nested_files(t_yml, dirpath))
elif t == "OS::Heat::ResourceGroup":
rdt = v.get("properties", {}).get("resource_def", {}).get("type", None)
if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")):
filepath = path.join(dirpath, rdt)
if path.exists(filepath):
with open(filepath) as fh:
rdt_yml = yaml.load(fh)
nested_files.append(filepath)
nested_files.extend(get_list_of_nested_files(rdt_yml, dirpath))
if isinstance(v, dict):
nested_files.extend(get_list_of_nested_files(v, dirpath))
elif isinstance(v, list):
for d in v:
nested_files.extend(get_list_of_nested_files(d, dirpath))
return nested_files
def get_nesting(yaml_files):
"""return bad, files, heat, depths
bad - list of error messages.
files - dict: key is filename, value is dict of nested files.
This is the tree.
heat - dict,: key is filename, value is Heat instance.
depths - dict: key is filename, value is a depth tuple
level: 0 1 2 3
file: template -> nested -> nested -> nested
depth: 3 2 1 0
"""
bad = []
files = {}
heat = {}
depths = {}
for yaml_file in yaml_files:
dirname, basename = path.split(yaml_file)
h = Heat(filepath=yaml_file)
heat[basename] = h
files[basename] = get_dict_of_nested_files(h.yml, dirname)
for filename in files:
depths[filename] = _get_nesting_depth_start(0, filename, files, [])
for depth in depths[filename]:
if depth[0] > MAX_DEPTH:
bad.append("{} {}".format(filename, str(depth[1])))
return bad, files, heat, depths
def _get_nesting_depth_start(depth, filename, files, context):
depths = []
for rid, nf in files[filename].items():
depths.append(_get_nesting_depth(1, nf, files, context))
return depths
def _get_nesting_depth(depth, filename, files, context):
"""Return a depth tuple (max_depth, current_context).
`context` is the list of filenames.
`depth` is the length of `context`.
Finds the max_depth of all the resources of `filename`.
current_context is the updated list of filenames
and max_depth is its length.
"""
max_depth = depth + 1
current_context = context + [filename]
if depth <= MAX_DEPTH:
nested_filenames = files.get(filename, {})
if nested_filenames:
max_depth, current_context = max(
_get_nesting_depth(depth + 1, nested_filename, files, current_context)
for nested_filename in nested_filenames.values()
)
return max_depth, current_context
def get_resourcegroup_nested_files(yml, dirpath):
"""
return a dict.
key: key in yml which references a nested ResourceGroup file.
(resource->type is ResourceGroup
and resource->properties->resource_def->type is a yaml file)
value: the nested file name.
The keys are assumed to be unique across files.
A separate test checks for that.
"""
if not hasattr(yml, "get"):
return {}
nested_files = {}
for rid, r in yml.get("resources", {}).items():
if isinstance(r, dict) and "type" in r:
t = r["type"]
nested_file = None
if t == "OS::Heat::ResourceGroup":
rdt = r.get("properties", {}).get("resource_def", {}).get("type", None)
if rdt and (rdt.endswith(".yml") or rdt.endswith(".yaml")):
nested_file = rdt
if nested_file:
filepath = path.join(dirpath, nested_file)
if path.exists(filepath):
nested_files[rid] = nested_file
return nested_files
def get_type_nested_files(yml, dirpath):
"""
return a dict.
key: key in yml which references a nested type file.
(the resource "type" is a yaml file.)
value: the nested file name.
The keys are assumed to be unique across files.
A separate test checks for that.
"""
if not hasattr(yml, "get"):
return {}
nested_files = {}
for rid, r in yml.get("resources", {}).items():
if isinstance(r, dict) and "type" in r:
t = r["type"]
nested_file = None
if t.endswith(".yml") or t.endswith(".yaml"):
nested_file = t
if nested_file:
filepath = path.join(dirpath, nested_file)
if path.exists(filepath):
nested_files[rid] = nested_file
return nested_files