diff options
Diffstat (limited to 'gen_requirement_changes.py')
| -rw-r--r-- | gen_requirement_changes.py | 312 |
1 files changed, 312 insertions, 0 deletions
diff --git a/gen_requirement_changes.py b/gen_requirement_changes.py new file mode 100644 index 0000000..2661c59 --- /dev/null +++ b/gen_requirement_changes.py @@ -0,0 +1,312 @@ +# -*- coding: utf8 -*- +# org.onap.vnfrqts/requirements +# ============LICENSE_START==================================================== +# Copyright © 2018 AT&T Intellectual Property. All rights reserved. +# +# Unless otherwise specified, all software contained herein is licensed +# under the Apache License, Version 2.0 (the "License"); +# you may not use this software except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Unless otherwise specified, all documentation contained herein is licensed +# under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +# you may not use this documentation except in compliance with the License. +# You may obtain a copy of the License at +# +# https://creativecommons.org/licenses/by/4.0/ +# +# Unless required by applicable law or agreed to in writing, documentation +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============LICENSE_END============================================ + +""" +This script will generate an summary of the requirements changes between +two version's of requirements by analyzing the needs.json file. The template +can be customized by updating release-requirement-changes.rst.jinja2. +""" +from itertools import groupby, chain +import json +import os +import re +import sys +import argparse +from operator import itemgetter + +import jinja2 + +REQ_JSON_URL = "https://onap.readthedocs.io/en/latest/_downloads/needs.json" +NEEDS_PATH = "docs/data/needs.json" +JINJA_TEMPLATE = "release-requirement-changes.rst.jinja2" + + +def check(predicate, msg): + """ + Raises a ``RuntimeError`` if the given predicate is false. + + :param predicate: Predicate to evaluate + :param msg: Error message to use if predicate is false + """ + if not predicate: + raise RuntimeError(msg) + + +class DifferenceFinder: + """ + Class takes a needs.json data structure and finds the differences + between two different versions of the requirements + """ + + def __init__(self, needs, current_version, prior_version): + """ + Determine the differences between the ``current_version`` and the + ``prior_version`` of the given requirements. + + :param needs: previously loaded needs.json file + :param current_version: most recent version to compare against + :param prior_version: a prior version + :return: + """ + self.needs = needs + self.current_version = current_version + self.prior_version = prior_version + self._validate() + + def _validate(self): + """ + Validates the inputs to the ``DifferenceFinder`` constructor. + + :raises RuntimeError: if the file is not structured properly or the + given versions can't be found. + """ + check(self.needs is not None, "needs cannot be None") + check(isinstance(self.needs, dict), "needs must be be a dict") + check("versions" in self.needs, "needs file not properly formatted") + for version in (self.current_version, self.prior_version): + check( + version in self.needs["versions"], + "Version " + version + " was not found in the needs file", + ) + + @property + def current_requirements(self): + """Returns a dict of requirement ID to requirement metadata""" + return self.get_version(self.current_version) + + @property + def prior_requirements(self): + """Returns a dict of requirement ID to requirement metadata""" + return self.get_version(self.prior_version) + + def get_version(self, version): + """Returns a dict of requirement ID to requirement metadata""" + return self.needs["versions"][version]["needs"] + + @property + def new_requirements(self): + """Requirements added since the prior version""" + new_ids = self.current_ids.difference(self.prior_ids) + return self.filter_needs(self.current_requirements, new_ids) + + @property + def current_ids(self): + """Returns a set of the requirement IDs for the current version""" + return set(self.current_requirements.keys()) + + @property + def prior_ids(self): + """Returns a set of the requirement IDs for the prior version""" + return set(self.prior_requirements.keys()) + + @property + def removed_requirements(self): + """Requirements that were removed since the prior version""" + removed_ids = self.prior_ids.difference(self.current_ids) + return self.filter_needs(self.prior_requirements, removed_ids) + + @property + def changed_requirements(self): + """"Requirements where the description changed since the last version""" + common_ids = self.prior_ids.intersection(self.current_ids) + result = {} + for r_id in common_ids: + current_text = self.current_requirements[r_id]["description"] + prior_text = self.prior_requirements[r_id]["description"] + if not self.is_equivalent(current_text, prior_text): + sections = self.current_requirements[r_id]["sections"] + result[r_id] = { + "id": r_id, + "description": current_text, + "sections": sections, + } + return result + + def is_equivalent(self, current_text, prior_text): + """Returns true if there are meaningful differences between the + text. See normalize for more information""" + return self.normalize(current_text) == self.normalize(prior_text) + + @staticmethod + def normalize(text): + """Strips out formatting, line breaks, and repeated spaces to normalize + the string for comparison. This ensures minor formatting changes + are not tagged as meaningful changes""" + s = text.lower() + s = s.replace("\n", " ") + s = re.sub(r'[`*\'"]', "", s) + s = re.sub(r"\s+", " ", s) + return s + + @staticmethod + def filter_needs(needs, ids): + """ + Return the requirements with the given ids + + :ids: sequence of requirement IDs + :returns: dict of requirement ID to requirement data for only the + requirements in ``ids`` + """ + return {r_id: data for r_id, data in needs.items() if r_id in ids} + + +def load_requirements(path): + """Load the requirements from the needs.json file""" + if not (os.path.exists(path)): + print("needs.json not found. Run tox -e docs to generate it.") + sys.exit(1) + with open(path, "r") as req_file: + return json.load(req_file) + + +def parse_args(): + """Parse the command-line arguments and return the arguments: + + args.current_version + args.prior_version + """ + parser = argparse.ArgumentParser( + description="Generate RST summarizing requirement changes between " + "two given releases. The resulting RST file will be " + "written to the docs/ directory" + ) + parser.add_argument( + "current_version", help="Current release in lowercase (ex: casablanca)" + ) + parser.add_argument("prior_version", + help="Prior release to compare against") + return parser.parse_args() + + +def tag(dicts, key, value): + """Adds the key value to each dict in the sequence""" + for d in dicts: + d[key] = value + return dicts + + +def gather_section_changes(diffs): + """ + Return a list of dicts that represent the changes for a given section path. + [ + { + "section_path": path, + "added: [req, ...], + "updated: [req, ...], + "removed: [req, ...], + }, + ... + ] + :param diffs: instance of DifferenceFinder + :return: list of section changes + """ + # Add "change" and "section_path" keys to all requirements + reqs = list( + chain( + tag(diffs.new_requirements.values(), "change", "added"), + tag(diffs.changed_requirements.values(), "change", "updated"), + tag(diffs.removed_requirements.values(), "change", "removed"), + ) + ) + for req in reqs: + req["section_path"] = " > ".join(reversed(req["sections"])) + + # Build list of changes by section + reqs = sorted(reqs, key=itemgetter("section_path")) + all_changes = [] + for section, section_reqs in groupby(reqs, key=itemgetter("section_path")): + change = itemgetter("change") + section_reqs = sorted(section_reqs, key=change) + section_changes = {"section_path": section} + for change, change_reqs in groupby(section_reqs, key=change): + section_changes[change] = list(change_reqs) + if any(k in section_changes for k in ("added", "updated", "removed")): + all_changes.append(section_changes) + return all_changes + + +def render_to_file(template_path, output_path, **context): + """Read the template and render it ot the output_path using the given + context variables""" + with open(template_path, "r") as infile: + t = jinja2.Template(infile.read()) + result = t.render(**context) + with open(output_path, "w") as outfile: + outfile.write(result) + print() + print("Writing requirement changes to " + output_path) + print( + "Please be sure to update the docs/release-notes.rst document " + "with a link to this document" + ) + + +def print_invalid_metadata_report(difference_finder, current_version): + """Write a report to the console for any instances where differences + are detected, but teh appropriate :introduced: or :updated: metadata + is not applied to the requirement.""" + print("Validating Metadata...") + print() + print("Requirements Added, but Missing :introduced: Attribute") + print("----------------------------------------------------") + for req in difference_finder.new_requirements.values(): + if "introduced" not in req or req["introduced"] != current_version: + print(req["id"]) + print() + print("Requirements Changed, but Missing :updated: Attribute") + print("-----------------------------------------------------") + for req in difference_finder.changed_requirements.values(): + if "updated" not in req or req["updated"] != current_version: + print(req["id"]) + + +if __name__ == "__main__": + args = parse_args() + requirements = load_requirements(NEEDS_PATH) + differ = DifferenceFinder(requirements, + args.current_version, + args.prior_version) + + print_invalid_metadata_report(differ, args.current_version) + + changes = gather_section_changes(differ) + render_to_file( + "release-requirement-changes.rst.jinja2", + "docs/changes-by-section-" + args.current_version + ".rst", + changes=changes, + current_version=args.current_version, + prior_version=args.prior_version, + num_added=len(differ.new_requirements), + num_removed=len(differ.removed_requirements), + num_changed=len(differ.changed_requirements), + ) |