1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
# -*- coding: utf8 -*-
# ============LICENSE_START====================================================
# org.onap.vvp/validation-scripts
# ===================================================================
# Copyright © 2019 AT&T Intellectual Property. All rights reserved.
# ===================================================================
#
# Unless otherwise specified, all software contained herein is licensed
# under the Apache License, Version 2.0 (the "License");
# you may not use this software except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
#
# Unless otherwise specified, all documentation contained herein is licensed
# under the Creative Commons License, Attribution 4.0 Intl. (the "License");
# you may not use this documentation except in compliance with the License.
# You may obtain a copy of the License at
#
# https://creativecommons.org/licenses/by/4.0/
#
# Unless required by applicable law or agreed to in writing, documentation
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ============LICENSE_END============================================
#
import contextlib
import csv
import io
import json
import os
import subprocess #nosec
import sys
import pytest
from flake8.main.application import Application
from update_reqs import get_requirements
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
CURRENT_NEEDS_PATH = os.path.join(THIS_DIR, "ice_validator/heat_requirements.json")
class Traceability:
PATH = os.path.join(THIS_DIR, "ice_validator/output/traceability.csv")
TEST_FILE = 6
TEST_NAME = 7
IS_TESTABLE = 5
REQ_ID = 0
def __init__(self):
with open(self.PATH, "r") as f:
rows = csv.reader(f)
next(rows) # skip header
self.mappings = list(rows)
def unmapped_requirement_errors(self):
"""
Returns list of errors where a requirement is testable, but no test was found.
"""
testable_mappings = [m for m in self.mappings if m[self.IS_TESTABLE] == "True"]
return [
f"Missing test for {m[self.REQ_ID]}"
for m in testable_mappings
if not m[self.TEST_NAME]
]
def mapped_non_testable_requirement_errors(self):
"""
Returns list of errors where the requirement isn't testable, but a test was
found.
"""
non_testables = [m for m in self.mappings if m[self.IS_TESTABLE] == "False"]
return [
(
f"No test for {m[0]} is needed, but found: "
f"{m[self.TEST_FILE]}::{m[self.TEST_NAME]} "
)
for m in non_testables
if m[self.TEST_NAME]
]
def current_version(needs):
"""Extracts and returns the needs under the current version"""
return needs["versions"][needs["current_version"]]["needs"]
def in_scope(_, req_metadata):
"""
Checks if requirement is relevant to VVP.
:param: _: not used
:param req_metadata: needs metadata about the requirement
:return: True if the requirement is a testable, Heat requirement
"""
return (
"Heat" in req_metadata.get("docname", "")
and "MUST" in req_metadata.get("keyword", "").upper()
and req_metadata.get("validation_mode", "").lower() != "none"
)
def select_items(predicate, source_dict):
"""
Creates a new dict from the source dict where the items match the given predicate
:param predicate: predicate function that must accept a two arguments (key & value)
:param source_dict: input dictionary to select from
:return: filtered dict
"""
return {k: v for k, v in source_dict.items() if predicate(k, v)}
def check_requirements_up_to_date():
"""
Checks if the requirements file packaged with VVP has meaningful differences
to the requirements file published from VNFRQTS.
:return: list of errors found
"""
msg = ["heat_requirements.json is out-of-date. Run update_reqs.py to update."]
latest_needs = json.load(get_requirements())
with open(CURRENT_NEEDS_PATH, "r") as f:
current_needs = json.load(f)
latest_reqs = select_items(in_scope, current_version(latest_needs))
current_reqs = select_items(in_scope, current_version(current_needs))
if set(latest_reqs.keys()) != set(current_reqs.keys()):
return msg
if not all(
latest["description"] == current_reqs[r_id]["description"]
for r_id, latest in latest_reqs.items()
):
return msg
return None
def check_self_test_pass():
"""
Run pytest self-test and ensure it passes
:return:
"""
original_dir = os.getcwd()
try:
os.chdir(os.path.join(THIS_DIR, "ice_validator"))
if pytest.main(["tests", "--self-test"]) != 0:
return ["VVP self-test failed. Run pytest --self-test and fix errors."]
finally:
os.chdir(original_dir)
def check_testable_requirements_are_mapped():
tracing = Traceability()
return tracing.unmapped_requirement_errors()
def check_non_testable_requirements_are_not_mapped():
tracing = Traceability()
return tracing.mapped_non_testable_requirement_errors()
def check_flake8_passes():
output = io.StringIO()
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output):
app = Application()
app.run(["ice_validator"])
output.seek(0)
lines = [f" {l}" for l in output.readlines()]
return ["flake8 errors detected:"] + lines if lines else []
def check_bandit_passes():
result = subprocess.run( #nosec
["bandit", "-c", "bandit.yaml", "-r", ".", "-x", "./.tox/**"], #nosec
encoding="utf-8", #nosec
stdout=subprocess.PIPE, #nosec
stderr=subprocess.PIPE, #nosec
) #nosec
msgs = result.stdout.split("\n") if result.returncode != 0 else []
return ["bandit errors detected:"] + [f" {e}" for e in msgs] if msgs else []
if __name__ == "__main__":
checks = [
check_self_test_pass,
check_requirements_up_to_date,
check_testable_requirements_are_mapped,
check_non_testable_requirements_are_not_mapped,
check_flake8_passes,
check_bandit_passes,
]
results = [check() for check in checks]
errors = "\n".join("\n".join(msg) for msg in results if msg)
print(errors or "Everything looks good!")
sys.exit(1 if errors else 0)
|
