diff options
Diffstat (limited to 'fix_invalid_metadata.py')
| -rw-r--r-- | fix_invalid_metadata.py | 306 |
1 files changed, 306 insertions, 0 deletions
diff --git a/fix_invalid_metadata.py b/fix_invalid_metadata.py new file mode 100644 index 0000000..dbff72c --- /dev/null +++ b/fix_invalid_metadata.py @@ -0,0 +1,306 @@ +# -*- coding: utf8 -*- +# org.onap.vnfrqts/requirements +# ============LICENSE_START==================================================== +# Copyright © 2018 AT&T Intellectual Property. All rights reserved. +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the "License"); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ + +""" +This script will consume the `invalid_metadata.csv` file produced by +`gen_requirement_changes.py`, then add/update any `:introduced:` or `:updated:` +attributes that may be missing from req directives. +""" +import csv +import os +import re +from collections import OrderedDict + +import pytest + +INPUT_FILE = "invalid_metadata.csv" + + +def load_invalid_reqs(fileobj): + """Load the invalid requirements from the input file into a dict""" + reader = csv.reader(fileobj) + next(reader) # skip header + return {row[0]: (row[1].strip(), row[2].strip()) for row in reader} + + +def check(predicate, msg): + """Raises a RuntimeError with the given msg if the predicate is false""" + if not predicate: + raise RuntimeError(msg) + + +class MetadataFixer: + """Takes a dict of requirement ID to expected metadata value. The + NeedsVisitor will pass the requirement attributes as a a dict + to `__call__`. If the requirement is one that needs to be fixed, then + it will add or update the attributes as needed and return it to the + visitor, otherwise it will return the attributes unchanged.""" + + def __init__(self, reqs_to_fix): + """Initialize the fixer with a dict of requirement ID to tuple of + (attr name, attr value).""" + self.reqs_to_fix = reqs_to_fix + + def __call__(self, metadata): + """If metadata is for a requirement that needs to be fixed, then + adds or updates the attribute as needed and returns it, otherwise + it returns metadata unchanged.""" + r_id = metadata[":id:"] + if r_id in self.reqs_to_fix: + attr, value = self.reqs_to_fix[r_id] + metadata[attr] = value + return metadata + + +class NeedsVisitor: + """Walks a directory for reStructuredText files and detects needs + directives as defined by sphinxcontrib-needs. When a directive is + found, then attributes are passed to a callback for processing if the + callback returns a dict of attributes, then the revised dict is used + instead of the attributes that were passed""" + + def __init__(self, func): + self.directives = re.compile("\.\.\s+req::.*") + self.func = func + + def process(self, root_dir): + """Walks the `root_dir` looking for rst to files to parse""" + for dir_path, sub_dirs, filenames in os.walk(root_dir): + for filename in filenames: + if filename.lower().endswith(".rst"): + self.handle_rst_file(os.path.join(dir_path, filename)) + + @staticmethod + def read(path): + """Read file at `path` and return lines as list""" + with open(path, "r") as f: + print("path=", path) + return list(f) + + @staticmethod + def write(path, content): + """Write a content to the given path""" + with open(path, "w") as f: + for line in content: + f.write(line) + + def handle_rst_file(self, path): + lines = (line for line in self.read(path)) + new_contents = [] + for line in lines: + if self.is_needs_directive(line): + metadata_lines = self.handle_need(lines) + new_contents.append(line) + new_contents.extend(metadata_lines) + else: + new_contents.append(line) + self.write(path, new_contents) + + def is_needs_directive(self, line): + """Returns True if the line denotes the start of a needs directive""" + return bool(self.directives.match(line)) + + def handle_need(self, lines): + """Called when a needs directive is encountered. The lines + will be positioned on the line after the directive. The attributes + will be read, and then passed to the visitor for processing""" + attributes = OrderedDict() + indent = 4 + for line in lines: + if line.strip().startswith(":"): + indent = self.calc_indent(line) + attr, value = self.parse_attribute(line) + attributes[attr] = value + else: + if attributes: + new_attributes = self.func(attributes) + attr_lines = self.format_attributes(new_attributes, indent) + return attr_lines + [line] + else: + return [line] + + @staticmethod + def format_attributes(attrs, indent): + """Converts a dict back to properly indented lines""" + spaces = " " * indent + return ["{}{} {}\n".format(spaces, k, v) for k, v in attrs.items()] + + @staticmethod + def parse_attribute(line): + return re.split("\s+", line.strip(), maxsplit=1) + + @staticmethod + def calc_indent(line): + return len(line) - len(line.lstrip()) + + +if __name__ == '__main__': + with open(INPUT_FILE, "r") as f: + invalid_reqs = load_invalid_reqs(f) + metadata_fixer = MetadataFixer(invalid_reqs) + visitor = NeedsVisitor(metadata_fixer) + visitor.process("docs") + + +# Tests +@pytest.fixture +def metadata_fixer(): + fixes = { + "R-1234": (":introduced:", "casablanca"), + "R-5678": (":updated:", "casablanca"), + } + return MetadataFixer(fixes) + + +def test_check_raises_when_false(): + with pytest.raises(RuntimeError): + check(False, "error") + + +def test_check_does_not_raise_when_true(): + check(True, "error") + + +def test_load_invalid_req(): + contents = [ + "reqt_id, attribute, value", + "R-1234,:introduced:, casablanca", + "R-5678,:updated:, casablanca", + ] + result = load_invalid_reqs(contents) + assert len(result) == 2 + assert result["R-1234"][0] == ":introduced:" + assert result["R-1234"][1] == "casablanca" + + +def test_metadata_fixer_adds_when_missing(metadata_fixer): + attributes = {":id:": "R-5678", ":introduced:": "beijing"} + result = metadata_fixer(attributes) + assert ":updated:" in result + assert result[":updated:"] == "casablanca" + + +def test_metadata_fixer_updates_when_incorrect(metadata_fixer): + attributes = {":id:": "R-5678", ":updated:": "beijing"} + result = metadata_fixer(attributes) + assert ":updated:" in result + assert result[":updated:"] == "casablanca" + assert ":introduced:" not in result + + +def test_needs_visitor_process(monkeypatch): + v = NeedsVisitor(lambda x: x) + paths = [] + + def mock_handle_rst(path): + paths.append(path) + + monkeypatch.setattr(v, "handle_rst_file", mock_handle_rst) + v.process("docs") + + assert len(paths) > 1 + assert all(path.endswith(".rst") for path in paths) + + +def test_needs_visitor_is_needs_directive(): + v = NeedsVisitor(lambda x: x) + assert v.is_needs_directive(".. req::") + assert not v.is_needs_directive("test") + assert not v.is_needs_directive(".. code::") + + +def test_needs_visitor_format_attributes(): + v = NeedsVisitor(lambda x: x) + attr = OrderedDict() + attr[":id:"] = "R-12345" + attr[":updated:"] = "casablanca" + lines = v.format_attributes(attr, 4) + assert len(lines) == 2 + assert lines[0] == " :id: R-12345" + assert lines[1] == " :updated: casablanca" + + +def test_needs_visitor_parse_attributes(): + v = NeedsVisitor(lambda x: x) + assert v.parse_attribute(" :id: R-12345") == [":id:", "R-12345"] + assert v.parse_attribute(" :key: one two") == [":key:", "one two"] + + +def test_needs_visitor_calc_indent(): + v = NeedsVisitor(lambda x: x) + assert v.calc_indent(" test") == 4 + assert v.calc_indent(" test") == 3 + assert v.calc_indent("test") == 0 + + +def test_needs_visitor_no_change(monkeypatch): + v = NeedsVisitor(lambda x: x) + lines = """.. req:: + :id: R-12345 + :updated: casablanca + + Here's my requirement""" + monkeypatch.setattr(v, "read", lambda path: lines.split("\n")) + result = [] + monkeypatch.setattr(v, "write", lambda _, content: result.extend(content)) + + v.handle_rst_file("dummy_path") + assert len(result) == 5 + assert "\n".join(result) == lines + + +def test_needs_visitor_with_fix(monkeypatch): + fixer = MetadataFixer({"R-12345": (":introduced:", "casablanca")}) + v = NeedsVisitor(fixer) + lines = """.. req:: + :id: R-12345 + + Here's my requirement""" + monkeypatch.setattr(v, "read", lambda path: lines.split("\n")) + result = [] + monkeypatch.setattr(v, "write", lambda _, content: result.extend(content)) + + v.handle_rst_file("dummy_path") + assert len(result) == 5 + assert ":introduced: casablanca" in "\n".join(result) + + +def test_load_invalid_reqs(): + input_file = [ + "r_id,attr,value", + "R-12345,:updated:,casablanca" + ] + result = load_invalid_reqs(input_file) + assert "R-12345" in result + assert result["R-12345"][0] == ":updated:" + assert result["R-12345"][1] == "casablanca" |