Requirement ID Generation and RST Validation

The new check.py script will now perform a variety of
actions to simplify updates and ensure specific
practices are followed for each update.

The script has been integrated with tox and
will run whenever the documentation is created.
It can also be ran separately by just invoking
python check.py.

The script will perform a variety of automatic
updates where possible, and provide a warning
where auto-updates are not possible.

The expecation is that all warnings are addressed
before submitting for review, but given it is a
new feature warnings do not block validation at
this time.

Here is a summary of the warnings and updates:

Warnings:
- Requirement missing required attributes
- Invalid values for attributes
- Invalid section header usage in any file
- :keyword: and requirement mismatch

Auto Updates:
- Assigning :id: on new requirements where an ID missing
- Adding :introduced: attribute on new requirements
- Adding/correcting :updated: attribute on changed requirements

Issue-ID: VNFRQTS-896
Signed-off-by: Lovett, Trevor <trevor.lovett@att.com>
Change-Id: I283441330a139aa1c6e2e79f0c54c5979bf44642

1 files changed, 0 insertions, 306 deletions

diff --git a/fix_invalid_metadata.py b/fix_invalid_metadata.py
deleted file mode 100644
index dbff72c..0000000
--- a/fix_invalid_metadata.py
+++ /dev/null
@@ -1,306 +0,0 @@
-# -*- coding: utf8 -*-
-# org.onap.vnfrqts/requirements
-# ============LICENSE_START====================================================
-# Copyright © 2018 AT&T Intellectual Property. All rights reserved.
-#
-# Unless otherwise specified, all software contained herein is licensed
-# under the Apache License, Version 2.0 (the "License");
-# you may not use this software except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#             http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Unless otherwise specified, all documentation contained herein is licensed
-# under the Creative Commons License, Attribution 4.0 Intl. (the "License");
-# you may not use this documentation except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#             https://creativecommons.org/licenses/by/4.0/
-#
-# Unless required by applicable law or agreed to in writing, documentation
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# ============LICENSE_END============================================
-
-"""
-This script will consume the `invalid_metadata.csv` file produced by
-`gen_requirement_changes.py`, then add/update any `:introduced:` or `:updated:`
-attributes that may be missing from req directives.
-"""
-import csv
-import os
-import re
-from collections import OrderedDict
-
-import pytest
-
-INPUT_FILE = "invalid_metadata.csv"
-
-
-def load_invalid_reqs(fileobj):
-    """Load the invalid requirements from the input file into a dict"""
-    reader = csv.reader(fileobj)
-    next(reader)  # skip header
-    return {row[0]: (row[1].strip(), row[2].strip()) for row in reader}
-
-
-def check(predicate, msg):
-    """Raises a RuntimeError with the given msg if the predicate is false"""
-    if not predicate:
-        raise RuntimeError(msg)
-
-
-class MetadataFixer:
-    """Takes a dict of requirement ID to expected metadata value.  The
-    NeedsVisitor will pass the requirement attributes as a a dict
-    to `__call__`.  If the requirement is one that needs to be fixed, then
-    it will add or update the attributes as needed and return it to the
-    visitor, otherwise it will return the attributes unchanged."""
-
-    def __init__(self, reqs_to_fix):
-        """Initialize the fixer with a dict of requirement ID to tuple of
-        (attr name, attr value)."""
-        self.reqs_to_fix = reqs_to_fix
-
-    def __call__(self, metadata):
-        """If metadata is for a requirement that needs to be fixed, then
-        adds or updates the attribute as needed and returns it, otherwise
-        it returns metadata unchanged."""
-        r_id = metadata[":id:"]
-        if r_id in self.reqs_to_fix:
-            attr, value = self.reqs_to_fix[r_id]
-            metadata[attr] = value
-        return metadata
-
-
-class NeedsVisitor:
-    """Walks a directory for reStructuredText files and detects needs
-    directives as defined by sphinxcontrib-needs.  When a directive is
-    found, then attributes are passed to a callback for processing if the
-    callback returns a dict of attributes, then the revised dict is used
-    instead of the attributes that were passed"""
-
-    def __init__(self, func):
-        self.directives = re.compile("\.\.\s+req::.*")
-        self.func = func
-
-    def process(self, root_dir):
-        """Walks the `root_dir` looking for rst to files to parse"""
-        for dir_path, sub_dirs, filenames in os.walk(root_dir):
-            for filename in filenames:
-                if filename.lower().endswith(".rst"):
-                    self.handle_rst_file(os.path.join(dir_path, filename))
-
-    @staticmethod
-    def read(path):
-        """Read file at `path` and return lines as list"""
-        with open(path, "r") as f:
-            print("path=", path)
-            return list(f)
-
-    @staticmethod
-    def write(path, content):
-        """Write a content to the given path"""
-        with open(path, "w") as f:
-            for line in content:
-                f.write(line)
-
-    def handle_rst_file(self, path):
-        lines = (line for line in self.read(path))
-        new_contents = []
-        for line in lines:
-            if self.is_needs_directive(line):
-                metadata_lines = self.handle_need(lines)
-                new_contents.append(line)
-                new_contents.extend(metadata_lines)
-            else:
-                new_contents.append(line)
-        self.write(path, new_contents)
-
-    def is_needs_directive(self, line):
-        """Returns True if the line denotes the start of a needs directive"""
-        return bool(self.directives.match(line))
-
-    def handle_need(self, lines):
-        """Called when a needs directive is encountered.  The lines
-        will be positioned on the line after the directive.  The attributes
-        will be read, and then passed to the visitor for processing"""
-        attributes = OrderedDict()
-        indent = 4
-        for line in lines:
-            if line.strip().startswith(":"):
-                indent = self.calc_indent(line)
-                attr, value = self.parse_attribute(line)
-                attributes[attr] = value
-            else:
-                if attributes:
-                    new_attributes = self.func(attributes)
-                    attr_lines = self.format_attributes(new_attributes, indent)
-                    return attr_lines + [line]
-                else:
-                    return [line]
-
-    @staticmethod
-    def format_attributes(attrs, indent):
-        """Converts a dict back to properly indented lines"""
-        spaces = " " * indent
-        return ["{}{} {}\n".format(spaces, k, v) for k, v in attrs.items()]
-
-    @staticmethod
-    def parse_attribute(line):
-        return re.split("\s+", line.strip(), maxsplit=1)
-
-    @staticmethod
-    def calc_indent(line):
-        return len(line) - len(line.lstrip())
-
-
-if __name__ == '__main__':
-    with open(INPUT_FILE, "r") as f:
-        invalid_reqs = load_invalid_reqs(f)
-    metadata_fixer = MetadataFixer(invalid_reqs)
-    visitor = NeedsVisitor(metadata_fixer)
-    visitor.process("docs")
-
-
-# Tests
-@pytest.fixture
-def metadata_fixer():
-    fixes = {
-        "R-1234": (":introduced:", "casablanca"),
-        "R-5678": (":updated:", "casablanca"),
-    }
-    return MetadataFixer(fixes)
-
-
-def test_check_raises_when_false():
-    with pytest.raises(RuntimeError):
-        check(False, "error")
-
-
-def test_check_does_not_raise_when_true():
-    check(True, "error")
-
-
-def test_load_invalid_req():
-    contents = [
-        "reqt_id, attribute, value",
-        "R-1234,:introduced:, casablanca",
-        "R-5678,:updated:, casablanca",
-    ]
-    result = load_invalid_reqs(contents)
-    assert len(result) == 2
-    assert result["R-1234"][0] == ":introduced:"
-    assert result["R-1234"][1] == "casablanca"
-
-
-def test_metadata_fixer_adds_when_missing(metadata_fixer):
-    attributes = {":id:": "R-5678", ":introduced:": "beijing"}
-    result = metadata_fixer(attributes)
-    assert ":updated:" in result
-    assert result[":updated:"] == "casablanca"
-
-
-def test_metadata_fixer_updates_when_incorrect(metadata_fixer):
-    attributes = {":id:": "R-5678", ":updated:": "beijing"}
-    result = metadata_fixer(attributes)
-    assert ":updated:" in result
-    assert result[":updated:"] == "casablanca"
-    assert ":introduced:" not in result
-
-
-def test_needs_visitor_process(monkeypatch):
-    v = NeedsVisitor(lambda x: x)
-    paths = []
-
-    def mock_handle_rst(path):
-        paths.append(path)
-
-    monkeypatch.setattr(v, "handle_rst_file", mock_handle_rst)
-    v.process("docs")
-
-    assert len(paths) > 1
-    assert all(path.endswith(".rst") for path in paths)
-
-
-def test_needs_visitor_is_needs_directive():
-    v = NeedsVisitor(lambda x: x)
-    assert v.is_needs_directive(".. req::")
-    assert not v.is_needs_directive("test")
-    assert not v.is_needs_directive(".. code::")
-
-
-def test_needs_visitor_format_attributes():
-    v = NeedsVisitor(lambda x: x)
-    attr = OrderedDict()
-    attr[":id:"] = "R-12345"
-    attr[":updated:"] = "casablanca"
-    lines = v.format_attributes(attr, 4)
-    assert len(lines) == 2
-    assert lines[0] == "    :id: R-12345"
-    assert lines[1] == "    :updated: casablanca"
-
-
-def test_needs_visitor_parse_attributes():
-    v = NeedsVisitor(lambda x: x)
-    assert v.parse_attribute("   :id: R-12345") == [":id:", "R-12345"]
-    assert v.parse_attribute("   :key: one two") == [":key:", "one two"]
-
-
-def test_needs_visitor_calc_indent():
-    v = NeedsVisitor(lambda x: x)
-    assert v.calc_indent("    test") == 4
-    assert v.calc_indent("   test") == 3
-    assert v.calc_indent("test") == 0
-
-
-def test_needs_visitor_no_change(monkeypatch):
-    v = NeedsVisitor(lambda x: x)
-    lines = """.. req::
-        :id: R-12345
-        :updated: casablanca
-        
-        Here's my requirement"""
-    monkeypatch.setattr(v, "read", lambda path: lines.split("\n"))
-    result = []
-    monkeypatch.setattr(v, "write", lambda _, content: result.extend(content))
-
-    v.handle_rst_file("dummy_path")
-    assert len(result) == 5
-    assert "\n".join(result) == lines
-
-
-def test_needs_visitor_with_fix(monkeypatch):
-    fixer = MetadataFixer({"R-12345": (":introduced:", "casablanca")})
-    v = NeedsVisitor(fixer)
-    lines = """.. req::
-        :id: R-12345
-
-        Here's my requirement"""
-    monkeypatch.setattr(v, "read", lambda path: lines.split("\n"))
-    result = []
-    monkeypatch.setattr(v, "write", lambda _, content: result.extend(content))
-
-    v.handle_rst_file("dummy_path")
-    assert len(result) == 5
-    assert ":introduced: casablanca" in "\n".join(result)
-
-
-def test_load_invalid_reqs():
-    input_file = [
-        "r_id,attr,value",
-        "R-12345,:updated:,casablanca"
-    ]
-    result = load_invalid_reqs(input_file)
-    assert "R-12345" in result
-    assert result["R-12345"][0] == ":updated:"
-    assert result["R-12345"][1] == "casablanca"